Detect n+1 issues involving Gitaly

.gitlab-ci.yml

@@ -128,7 +128,7 @@ stages:
     - export CACHE_CLASSES=true
     - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - scripts/gitaly-test-spawn
-    - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
+    - knapsack spinach "-r rerun" -b || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -b -r rerun $(cat tmp/spinach-rerun.txt)'
   artifacts:
     expire_in: 31d
     when: always

app/controllers/projects/branches_controller.rb

@@ -15,10 +15,14 @@ class Projects::BranchesController < Projects::ApplicationController
     respond_to do |format|
       format.html do
         @refs_pipelines = @project.pipelines.latest_successful_for_refs(@branches.map(&:name))
-        @max_commits = @branches.reduce(0) do |memo, branch|
-          diverging_commit_counts = repository.diverging_commit_counts(branch)
-          [memo, diverging_commit_counts[:behind], diverging_commit_counts[:ahead]].max
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37429
+        Gitlab::GitalyClient.allow_n_plus_1_calls do
+          @max_commits = @branches.reduce(0) do |memo, branch|
+            diverging_commit_counts = repository.diverging_commit_counts(branch)
+            [memo, diverging_commit_counts[:behind], diverging_commit_counts[:ahead]].max
+          end
+          render
         end
       end
       format.json do

app/controllers/projects/commit_controller.rb

@@ -20,7 +20,12 @@ class Projects::CommitController < Projects::ApplicationController
     apply_diff_view_cookie!
     respond_to do |format|
-      format.html
+      format.html do
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37599
+        Gitlab::GitalyClient.allow_n_plus_1_calls do
+          render
+        end
+      end
       format.diff  { render text: @commit.to_diff }
       format.patch { render text: @commit.to_patch }
     end

app/controllers/projects/compare_controller.rb

@@ -17,6 +17,10 @@ class Projects::CompareController < Projects::ApplicationController
   def show
     apply_diff_view_cookie!
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37430
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      render
+    end
   end
   def diff_for_path

app/controllers/projects/merge_requests/diffs_controller.rb

@@ -10,7 +10,10 @@ class Projects::MergeRequests::DiffsController < Projects::MergeRequests::Applic
   def show
     @environment = @merge_request.environments_for(current_user).last
-    render json: { html: view_to_html_string("projects/merge_requests/diffs/_diffs") }
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37431
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      render json: { html: view_to_html_string("projects/merge_requests/diffs/_diffs") }
+    end
   end
   def diff_for_path

app/controllers/projects/merge_requests_controller.rb

@@ -56,6 +56,9 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
     close_merge_request_without_source_project
     check_if_can_be_merged
+    # Return if the response has already been rendered
+    return if response_body
+
     respond_to do |format|
       format.html do
         # Build a note object for comment form
@@ -70,6 +73,11 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
         labels
         set_pipeline_variables
+
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37432
+        Gitlab::GitalyClient.allow_n_plus_1_calls do
+          render
+        end
       end
       format.json do

app/controllers/projects/network_controller.rb

@@ -8,19 +8,24 @@ class Projects::NetworkController < Projects::ApplicationController
   before_action :assign_commit
   def show
-    @url = project_network_path(@project, @ref, @options.merge(format: :json))
-    @commit_url = project_commit_path(@project, 'ae45ca32').gsub("ae45ca32", "%s")
-    respond_to do |format|
-      format.html do
-        if @options[:extended_sha1] && !@commit
-          flash.now[:alert] = "Git revision '#{@options[:extended_sha1]}' does not exist."
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37602
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      @url = project_network_path(@project, @ref, @options.merge(format: :json))
+      @commit_url = project_commit_path(@project, 'ae45ca32').gsub("ae45ca32", "%s")
+      respond_to do |format|
+        format.html do
+          if @options[:extended_sha1] && !@commit
+            flash.now[:alert] = "Git revision '#{@options[:extended_sha1]}' does not exist."
+          end
         end
-      end
-      format.json do
-        @graph = Network::Graph.new(project, @ref, @commit, @options[:filter_ref])
+        format.json do
+          @graph = Network::Graph.new(project, @ref, @commit, @options[:filter_ref])
+        end
       end
+
+      render
     end
   end

app/controllers/projects/refs_controller.rb

@@ -51,13 +51,16 @@ class Projects::RefsController < Projects::ApplicationController
     contents.push(*tree.blobs)
     contents.push(*tree.submodules)
-    @logs = contents[@offset, @limit].to_a.map do |content|
-      file = @path ? File.join(@path, content.name) : content.name
-      last_commit = @repo.last_commit_for_path(@commit.id, file)
-      {
-        file_name: content.name,
-        commit: last_commit
-      }
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37433
+    @logs = Gitlab::GitalyClient.allow_n_plus_1_calls do
+      contents[@offset, @limit].to_a.map do |content|
+        file = @path ? File.join(@path, content.name) : content.name
+        last_commit = @repo.last_commit_for_path(@commit.id, file)
+        {
+          file_name: content.name,
+          commit: last_commit
+        }
+      end
     end
     offset = (@offset + @limit)

app/controllers/root_controller.rb

@@ -13,7 +13,10 @@ class RootController < Dashboard::ProjectsController
   before_action :redirect_logged_user, if: -> { current_user.present? }
   def index
-    super
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37434
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      super
+    end
   end
   private

app/models/merge_request.rb

@@ -415,8 +415,11 @@ class MergeRequest < ActiveRecord::Base
   end
   def create_merge_request_diff
-    merge_request_diffs.create
-    reload_merge_request_diff
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37435
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      merge_request_diffs.create
+      reload_merge_request_diff
+    end
   end
   def reload_merge_request_diff

app/models/network/graph.rb

@@ -61,8 +61,11 @@ module Network
         @reserved[i] = []
       end
-      commits_sort_by_ref.each do |commit|
-        place_chain(commit)
+      # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37436
+      Gitlab::GitalyClient.allow_n_plus_1_calls do
+        commits_sort_by_ref.each do |commit|
+          place_chain(commit)
+        end
       end
       # find parent spaces for not overlap lines

app/models/repository.rb

@@ -983,6 +983,7 @@ class Repository
   def empty_repo?
     !exists? || !has_visible_content?
   end
+  cache_method :empty_repo?, memoize_only: true
   def search_files_by_content(query, ref)
     return [] if empty_repo? || query.blank?

app/services/delete_merged_branches_service.rb

@@ -6,15 +6,18 @@ class DeleteMergedBranchesService < BaseService
   def execute
     raise Gitlab::Access::AccessDeniedError unless can?(current_user, :push_code, project)
-    branches = project.repository.branch_names
-    branches = branches.select { |branch| project.repository.merged_to_root_ref?(branch) }
-    # Prevent deletion of branches relevant to open merge requests
-    branches -= merge_request_branch_names
-    # Prevent deletion of protected branches
-    branches = branches.reject { |branch| project.protected_for?(branch) }
-    branches.each do |branch|
-      DeleteBranchService.new(project, current_user).execute(branch)
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37438
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      branches = project.repository.branch_names
+      branches = branches.select { |branch| project.repository.merged_to_root_ref?(branch) }
+      # Prevent deletion of branches relevant to open merge requests
+      branches -= merge_request_branch_names
+      # Prevent deletion of protected branches
+      branches = branches.reject { |branch| project.protected_for?(branch) }
+      branches.each do |branch|
+        DeleteBranchService.new(project, current_user).execute(branch)
+      end
     end
   end

app/services/merge_requests/create_service.rb

@@ -13,7 +13,10 @@ module MergeRequests
       merge_request.source_branch = params[:source_branch]
       merge_request.merge_params['force_remove_source_branch'] = params.delete(:force_remove_source_branch)
-      create(merge_request)
+      # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37439
+      Gitlab::GitalyClient.allow_n_plus_1_calls do
+        create(merge_request)
+      end
     end
     def before_create(merge_request)

app/services/notes/create_service.rb

@@ -4,7 +4,13 @@ module Notes
       merge_request_diff_head_sha = params.delete(:merge_request_diff_head_sha)
       note = Notes::BuildService.new(project, current_user, params).execute
-      return note unless note.valid?
+
+      # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37440
+      note_valid = Gitlab::GitalyClient.allow_n_plus_1_calls do
+        note.valid?
+      end
+
+      return note unless note_valid
       # We execute commands (extracted from `params[:note]`) on the noteable
       # **before** we save the note because if the note consists of commands

config/initializers/lograge.rb

@@ -12,13 +12,18 @@ unless Sidekiq.server?
     config.lograge.logger = ActiveSupport::Logger.new(filename)
     # Add request parameters to log output
     config.lograge.custom_options = lambda do |event|
-      {
+      payload = {
         time: event.time.utc.iso8601(3),
         params: event.payload[:params].except(*%w(controller action format)),
         remote_ip: event.payload[:remote_ip],
         user_id: event.payload[:user_id],
         username: event.payload[:username]
       }
+
+      gitaly_calls = Gitlab::GitalyClient.get_request_count
+      payload[:gitaly_calls] = gitaly_calls if gitaly_calls > 0
+
+      payload
     end
   end
 end

lib/api/branches.rb

@@ -21,7 +21,10 @@ module API
       get ':id/repository/branches' do
         branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name))
-        present paginate(branches), with: Entities::RepoBranch, project: user_project
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442
+        Gitlab::GitalyClient.allow_n_plus_1_calls do
+          present paginate(branches), with: Entities::RepoBranch, project: user_project
+        end
       end
       resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do

lib/api/entities.rb

@@ -244,7 +244,10 @@ module API
       end
       expose :merged do |repo_branch, options|
-        options[:project].repository.merged_to_root_ref?(repo_branch.name)
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442
+        Gitlab::GitalyClient.allow_n_plus_1_calls do
+          options[:project].repository.merged_to_root_ref?(repo_branch.name)
+        end
       end
       expose :protected do |repo_branch, options|

lib/gitlab/diff/file_collection/base.rb

@@ -22,7 +22,10 @@ module Gitlab
         end
         def diff_files
-          @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+          # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37445
+          Gitlab::GitalyClient.allow_n_plus_1_calls do
+            @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+          end
         end
         def diff_file_with_old_path(old_path)

lib/gitlab/git/repository.rb

@@ -950,8 +950,8 @@ module Gitlab
         @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self)
       end
-      def gitaly_migrate(method, &block)
-        Gitlab::GitalyClient.migrate(method, &block)
+      def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
+        Gitlab::GitalyClient.migrate(method, status: status, &block)
       rescue GRPC::NotFound => e
         raise NoRepository.new(e)
       rescue GRPC::BadStatus => e
@@ -962,14 +962,17 @@ module Gitlab
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
       def branches_filter(filter: nil, sort_by: nil)
-        branches = rugged.branches.each(filter).map do |rugged_ref|
-          begin
-            target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
-            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
-          rescue Rugged::ReferenceError
-            # Omit invalid branch
-          end
-        end.compact
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464
+        branches = Gitlab::GitalyClient.allow_n_plus_1_calls do
+          rugged.branches.each(filter).map do |rugged_ref|
+            begin
+              target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+              Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+            rescue Rugged::ReferenceError
+              # Omit invalid branch
+            end
+          end.compact
+        end
         sort_branches(branches, sort_by)
       end