Merge branch 'gitlab-git-repository-log' into 'master'

Gitlab::Git shuffling and Gitaly annotations

See merge request !12849

app/models/repository.rb

@@ -123,6 +123,7 @@ class Repository
     commits
   end
+  # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/384
   def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset = 0)
     unless exists? && has_visible_content? && query.present?
       return []
@@ -610,6 +611,7 @@ class Repository
     commit(sha)
   end
+  # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/383
   def last_commit_id_for_path(sha, path)
     key = path.blank? ? "last_commit_id_for_path:#{sha}" : "last_commit_id_for_path:#{sha}:#{Digest::SHA1.hexdigest(path)}"

app/services/git_operation_service.rb

@@ -129,6 +129,7 @@ class GitOperationService
     end
   end
+  # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites.
   def update_ref(ref, newrev, oldrev)
     # We use 'git update-ref' because libgit2/rugged currently does not
     # offer 'compare and swap' ref updates. Without compare-and-swap we can

lib/gitlab/git/commit.rb

@@ -38,7 +38,7 @@ module Gitlab
           repo = options.delete(:repo)
           raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log)
-          repo.log(options).map { |c| decorate(c) }
+          repo.log(options)
         end
         # Get single commit

lib/gitlab/git/repository.rb

@@ -331,85 +331,10 @@ module Gitlab
       #   )
       #
       def log(options)
-        default_options = {
-          limit: 10,
-          offset: 0,
-          path: nil,
-          follow: false,
-          skip_merges: false,
-          disable_walk: false,
-          after: nil,
-          before: nil
-        }
-
-        options = default_options.merge(options)
-        options[:limit] ||= 0
-        options[:offset] ||= 0
-        actual_ref = options[:ref] || root_ref
-        begin
-          sha = sha_from_ref(actual_ref)
-        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
-          # Return an empty array if the ref wasn't found
-          return []
-        end
-
-        if log_using_shell?(options)
-          log_by_shell(sha, options)
-        else
-          log_by_walk(sha, options)
-        end
-      end
-
-      def log_using_shell?(options)
-        options[:path].present? ||
-          options[:disable_walk] ||
-          options[:skip_merges] ||
-          options[:after] ||
-          options[:before]
-      end
-
-      def log_by_walk(sha, options)
-        walk_options = {
-          show: sha,
-          sort: Rugged::SORT_NONE,
-          limit: options[:limit],
-          offset: options[:offset]
-        }
-        Rugged::Walker.walk(rugged, walk_options).to_a
-      end
-
-      def log_by_shell(sha, options)
-        limit = options[:limit].to_i
-        offset = options[:offset].to_i
-        use_follow_flag = options[:follow] && options[:path].present?
-
-        # We will perform the offset in Ruby because --follow doesn't play well with --skip.
-        # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
-        offset_in_ruby = use_follow_flag && options[:offset].present?
-        limit += offset if offset_in_ruby
-
-        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
-        cmd << "--max-count=#{limit}"
-        cmd << '--format=%H'
-        cmd << "--skip=#{offset}" unless offset_in_ruby
-        cmd << '--follow' if use_follow_flag
-        cmd << '--no-merges' if options[:skip_merges]
-        cmd << "--after=#{options[:after].iso8601}" if options[:after]
-        cmd << "--before=#{options[:before].iso8601}" if options[:before]
-        cmd << sha
-
-        # :path can be a string or an array of strings
-        if options[:path].present?
-          cmd << '--'
-          cmd += Array(options[:path])
-        end
-
-        raw_output = IO.popen(cmd) { |io| io.read }
-        lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
-
-        lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
+        raw_log(options).map { |c| Commit.decorate(c) }
       end
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382
       def count_commits(options)
         cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
         cmd << "--after=#{options[:after].iso8601}" if options[:after]
@@ -454,7 +379,7 @@ module Gitlab
       # Counts the amount of commits between `from` and `to`.
       def count_commits_between(from, to)
-        commits_between(from, to).size
+        Commit.between(self, from, to).size
       end
       # Returns the SHA of the most recent common ancestor of +from+ and +to+
@@ -912,6 +837,89 @@ module Gitlab
       private
+      def raw_log(options)
+        default_options = {
+          limit: 10,
+          offset: 0,
+          path: nil,
+          follow: false,
+          skip_merges: false,
+          disable_walk: false,
+          after: nil,
+          before: nil
+        }
+
+        options = default_options.merge(options)
+        options[:limit] ||= 0
+        options[:offset] ||= 0
+        actual_ref = options[:ref] || root_ref
+        begin
+          sha = sha_from_ref(actual_ref)
+        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+          # Return an empty array if the ref wasn't found
+          return []
+        end
+
+        if log_using_shell?(options)
+          log_by_shell(sha, options)
+        else
+          log_by_walk(sha, options)
+        end
+      end
+
+      def log_using_shell?(options)
+        options[:path].present? ||
+          options[:disable_walk] ||
+          options[:skip_merges] ||
+          options[:after] ||
+          options[:before]
+      end
+
+      def log_by_walk(sha, options)
+        walk_options = {
+          show: sha,
+          sort: Rugged::SORT_NONE,
+          limit: options[:limit],
+          offset: options[:offset]
+        }
+        Rugged::Walker.walk(rugged, walk_options).to_a
+      end
+
+      # Gitaly note: JV: although #log_by_shell shells out to Git I think the
+      # complexity is such that we should migrate it as Ruby before trying to
+      # do it in Go.
+      def log_by_shell(sha, options)
+        limit = options[:limit].to_i
+        offset = options[:offset].to_i
+        use_follow_flag = options[:follow] && options[:path].present?
+
+        # We will perform the offset in Ruby because --follow doesn't play well with --skip.
+        # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
+        offset_in_ruby = use_follow_flag && options[:offset].present?
+        limit += offset if offset_in_ruby
+
+        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
+        cmd << "--max-count=#{limit}"
+        cmd << '--format=%H'
+        cmd << "--skip=#{offset}" unless offset_in_ruby
+        cmd << '--follow' if use_follow_flag
+        cmd << '--no-merges' if options[:skip_merges]
+        cmd << "--after=#{options[:after].iso8601}" if options[:after]
+        cmd << "--before=#{options[:before].iso8601}" if options[:before]
+        cmd << sha
+
+        # :path can be a string or an array of strings
+        if options[:path].present?
+          cmd << '--'
+          cmd += Array(options[:path])
+        end
+
+        raw_output = IO.popen(cmd) { |io| io.read }
+        lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
+
+        lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
+      end
+
       # We are trying to deprecate this method because it does a lot of work
       # but it seems to be used only to look up submodule URL's.
       # https://gitlab.com/gitlab-org/gitaly/issues/329

lib/gitlab/git_ref_validator.rb

+# Gitaly note: JV: does not need to be migrated, works without a repo.
+
 module Gitlab
   module GitRefValidator
     extend self

lib/gitlab/shell.rb

+# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository.
+# SSH key operations are not part of Gitaly so will never be migrated.
+
 require 'securerandom'
 module Gitlab
@@ -68,6 +71,7 @@ module Gitlab
     # Ex.
     #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def add_repository(storage, name)
       gitlab_shell_fast_execute([gitlab_shell_projects_path,
                                  'add-project', storage, "#{name}.git"])
@@ -81,6 +85,7 @@ module Gitlab
     # Ex.
     #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def import_repository(storage, name, url)
       # Timeout should be less than 900 ideally, to prevent the memory killer
       # to silently kill the process without knowing we are timing out here.
@@ -99,6 +104,7 @@ module Gitlab
     # Ex.
     #   fetch_remote("gitlab/gitlab-ci", "upstream")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def fetch_remote(storage, name, remote, forced: false, no_tags: false)
       args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
       args << '--force' if forced
@@ -115,6 +121,7 @@ module Gitlab
     # Ex.
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def mv_repository(storage, path, new_path)
       gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
                                  storage, "#{path}.git", "#{new_path}.git"])
@@ -129,6 +136,7 @@ module Gitlab
     # Ex.
     #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx")
     #
+    # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
     def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace)
       gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project',
                                  forked_from_storage, "#{path}.git", forked_to_storage,
@@ -143,6 +151,7 @@ module Gitlab
     # Ex.
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def remove_repository(storage, name)
       gitlab_shell_fast_execute([gitlab_shell_projects_path,
                                  'rm-project', storage, "#{name}.git"])
@@ -194,6 +203,7 @@ module Gitlab
     # Ex.
     #   add_namespace("/path/to/storage", "gitlab")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def add_namespace(storage, name)
       path = full_path(storage, name)
       FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
@@ -207,6 +217,7 @@ module Gitlab
     # Ex.
     #   rm_namespace("/path/to/storage", "gitlab")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def rm_namespace(storage, name)
       FileUtils.rm_r(full_path(storage, name), force: true)
     end
@@ -216,6 +227,7 @@ module Gitlab
     # Ex.
     #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def mv_namespace(storage, old_name, new_name)
       return false if exists?(storage, new_name) || !exists?(storage, old_name)
@@ -241,6 +253,7 @@ module Gitlab
     #   exists?(storage, 'gitlab')
     #   exists?(storage, 'gitlab/cookies.git')
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def exists?(storage, dir_name)
       File.exist?(full_path(storage, dir_name))
     end

spec/lib/gitlab/git/repository_spec.rb

@@ -705,9 +705,9 @@ describe Gitlab::Git::Repository, seed_helper: true do
       # Add new commits so that there's a renamed file in the commit history
       repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged
-      commit_with_old_name = new_commit_edit_old_file(repo)
-      rename_commit = new_commit_move_file(repo)
-      commit_with_new_name = new_commit_edit_new_file(repo)
+      commit_with_old_name = Gitlab::Git::Commit.decorate(new_commit_edit_old_file(repo))
+      rename_commit = Gitlab::Git::Commit.decorate(new_commit_move_file(repo))
+      commit_with_new_name = Gitlab::Git::Commit.decorate(new_commit_edit_new_file(repo))
     end
     after(:context) do
@@ -880,8 +880,8 @@ describe Gitlab::Git::Repository, seed_helper: true do
     context "compare results between log_by_walk and log_by_shell" do
       let(:options) { { ref: "master" } }
-      let(:commits_by_walk) { repository.log(options).map(&:oid) }
-      let(:commits_by_shell) { repository.log(options.merge({ disable_walk: true })).map(&:oid) }
+      let(:commits_by_walk) { repository.log(options).map(&:id) }
+      let(:commits_by_shell) { repository.log(options.merge({ disable_walk: true })).map(&:id) }
       it { expect(commits_by_walk).to eq(commits_by_shell) }
@@ -924,7 +924,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
         expect(commits.size).to be > 0
         expect(commits).to satisfy do |commits|
-          commits.all? { |commit| commit.time >= options[:after] }
+          commits.all? { |commit| commit.committed_date >= options[:after] }
         end
       end
     end
@@ -937,7 +937,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
         expect(commits.size).to be > 0
         expect(commits).to satisfy do |commits|
-          commits.all? { |commit| commit.time <= options[:before] }
+          commits.all? { |commit| commit.committed_date <= options[:before] }
         end
       end
     end
@@ -946,7 +946,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
       let(:options) { { ref: 'master', path: ['PROCESS.md', 'README.md'] } }
       def commit_files(commit)
-        commit.diff(commit.parent_ids.first).deltas.flat_map do |delta|
+        commit.diff_from_parent.deltas.flat_map do |delta|
           [delta.old_file[:path], delta.new_file[:path]].uniq.compact
         end
       end