Track and act upon the number of executed queries

This ensures that we have more visibility in the number of SQL queries
that are executed in web requests. The current threshold is hardcoded to
100 as we will rarely (maybe once or twice) change it.

In production and development we use Sentry if enabled, in the test
environment we raise an error. This feature is also only enabled in
production/staging when running on GitLab.com as it's not very useful to
other users.

app/controllers/admin/services_controller.rb

 class Admin::ServicesController < Admin::ApplicationController
   include ServiceParams
+  before_action :whitelist_query_limiting, only: [:index]
   before_action :service, only: [:edit, :update]
   def index
@@ -37,4 +38,8 @@ class Admin::ServicesController < Admin::ApplicationController
   def service
     @service ||= Service.where(id: params[:id], template: true).first
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42430')
+  end
 end

app/controllers/boards/issues_controller.rb

@@ -2,6 +2,7 @@ module Boards
   class IssuesController < Boards::ApplicationController
     include BoardsResponses
+    before_action :whitelist_query_limiting, only: [:index, :update]
     before_action :authorize_read_issue, only: [:index]
     before_action :authorize_create_issue, only: [:create]
     before_action :authorize_update_issue, only: [:update]
@@ -92,5 +93,10 @@ module Boards
         }
       )
     end
+
+    def whitelist_query_limiting
+      # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42439
+      Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42428')
+    end
   end
 end

app/controllers/import/gitlab_projects_controller.rb

 class Import::GitlabProjectsController < Import::BaseController
+  before_action :whitelist_query_limiting, only: [:create]
   before_action :verify_gitlab_project_import_enabled
   def new
@@ -40,4 +41,8 @@ class Import::GitlabProjectsController < Import::BaseController
       :path, :namespace_id, :file
     )
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42437')
+  end
 end

app/controllers/projects/commits_controller.rb

@@ -4,6 +4,7 @@ class Projects::CommitsController < Projects::ApplicationController
   include ExtractsPath
   include RendersCommits
+  before_action :whitelist_query_limiting
   before_action :require_non_empty_project
   before_action :assign_ref_vars
   before_action :authorize_download_code!
@@ -65,4 +66,8 @@ class Projects::CommitsController < Projects::ApplicationController
     @commits = @commits.with_pipeline_status
     @commits = prepare_commits_for_rendering(@commits)
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42330')
+  end
 end

app/controllers/projects/cycle_analytics_controller.rb

@@ -3,6 +3,7 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController
   include ActionView::Helpers::TextHelper
   include CycleAnalyticsParams
+  before_action :whitelist_query_limiting, only: [:show]
   before_action :authorize_read_cycle_analytics!
   def show
@@ -31,4 +32,8 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController
       permissions: @cycle_analytics.permissions(user: current_user)
     }
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42671')
+  end
 end

app/controllers/projects/forks_controller.rb

@@ -2,6 +2,7 @@ class Projects::ForksController < Projects::ApplicationController
   include ContinueParams
   # Authorize
+  before_action :whitelist_query_limiting, only: [:create]
   before_action :require_non_empty_project
   before_action :authorize_download_code!
   before_action :authenticate_user!, only: [:new, :create]
@@ -54,4 +55,8 @@ class Projects::ForksController < Projects::ApplicationController
       render :error
     end
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42335')
+  end
 end

app/controllers/projects/issues_controller.rb

@@ -8,6 +8,7 @@ class Projects::IssuesController < Projects::ApplicationController
   prepend_before_action :authenticate_user!, only: [:new]
+  before_action :whitelist_query_limiting, only: [:create, :create_merge_request, :move, :bulk_update]
   before_action :check_issues_available!
   before_action :issue, except: [:index, :new, :create, :bulk_update]
   before_action :set_issuables_index, only: [:index]
@@ -247,4 +248,13 @@ class Projects::IssuesController < Projects::ApplicationController
     @finder_type = IssuesFinder
     super
   end
+
+  def whitelist_query_limiting
+    # Also see the following issues:
+    #
+    # 1. https://gitlab.com/gitlab-org/gitlab-ce/issues/42423
+    # 2. https://gitlab.com/gitlab-org/gitlab-ce/issues/42424
+    # 3. https://gitlab.com/gitlab-org/gitlab-ce/issues/42426
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42422')
+  end
 end

app/controllers/projects/merge_requests/creations_controller.rb

@@ -4,6 +4,7 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap
   include RendersCommits
   skip_before_action :merge_request
+  before_action :whitelist_query_limiting, only: [:create]
   before_action :authorize_create_merge_request!
   before_action :apply_diff_view_cookie!, only: [:diffs, :diff_for_path]
   before_action :build_merge_request, except: [:create]
@@ -125,4 +126,8 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap
       @project.forked_from_project
     end
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42384')
+  end
 end

app/controllers/projects/merge_requests_controller.rb

@@ -7,6 +7,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
   include IssuableCollections
   skip_before_action :merge_request, only: [:index, :bulk_update]
+  before_action :whitelist_query_limiting, only: [:assign_related_issues, :update]
   before_action :authorize_update_issuable!, only: [:close, :edit, :update, :remove_wip, :sort]
   before_action :set_issuables_index, only: [:index]
   before_action :authenticate_user!, only: [:assign_related_issues]
@@ -339,4 +340,9 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
     access_denied! unless access_check
   end
+
+  def whitelist_query_limiting
+    # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42441
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42438')
+  end
 end

app/controllers/projects/network_controller.rb

@@ -2,6 +2,7 @@ class Projects::NetworkController < Projects::ApplicationController
   include ExtractsPath
   include ApplicationHelper
+  before_action :whitelist_query_limiting
   before_action :require_non_empty_project
   before_action :assign_ref_vars
   before_action :authorize_download_code!
@@ -35,4 +36,8 @@ class Projects::NetworkController < Projects::ApplicationController
     @options[:extended_sha1] = params[:extended_sha1]
     @commit = @repo.commit(@options[:extended_sha1])
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42333')
+  end
 end

app/controllers/projects/notes_controller.rb

@@ -2,6 +2,7 @@ class Projects::NotesController < Projects::ApplicationController
   include NotesActions
   include ToggleAwardEmoji
+  before_action :whitelist_query_limiting, only: [:create]
   before_action :authorize_read_note!
   before_action :authorize_create_note!, only: [:create]
   before_action :authorize_resolve_note!, only: [:resolve, :unresolve]
@@ -79,4 +80,8 @@ class Projects::NotesController < Projects::ApplicationController
     access_denied! unless can?(current_user, :create_note, noteable)
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42383')
+  end
 end

app/controllers/projects/pipelines_controller.rb

 class Projects::PipelinesController < Projects::ApplicationController
+  before_action :whitelist_query_limiting, only: [:create, :retry]
   before_action :pipeline, except: [:index, :new, :create, :charts]
   before_action :commit, only: [:show, :builds, :failures]
   before_action :authorize_read_pipeline!
@@ -166,4 +167,9 @@ class Projects::PipelinesController < Projects::ApplicationController
   def commit
     @commit ||= @pipeline.commit
   end
+
+  def whitelist_query_limiting
+    # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42343
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42339')
+  end
 end

app/controllers/projects_controller.rb

@@ -3,6 +3,7 @@ class ProjectsController < Projects::ApplicationController
   include ExtractsPath
   include PreviewMarkdown
+  before_action :whitelist_query_limiting, only: [:create]
   before_action :authenticate_user!, except: [:index, :show, :activity, :refs]
   before_action :redirect_git_extension, only: [:show]
   before_action :project, except: [:index, :new, :create]
@@ -405,4 +406,8 @@ class ProjectsController < Projects::ApplicationController
     #
     redirect_to request.original_url.sub(%r{\.git/?\Z}, '') if params[:format] == 'git'
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
+  end
 end

app/controllers/registrations_controller.rb

 class RegistrationsController < Devise::RegistrationsController
   include Recaptcha::Verify
+  before_action :whitelist_query_limiting, only: [:destroy]
+
   def new
     redirect_to(new_user_session_path)
   end
@@ -83,4 +85,8 @@ class RegistrationsController < Devise::RegistrationsController
   def devise_mapping
     @devise_mapping ||= Devise.mappings[:user]
   end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42380')
+  end
 end

changelogs/unreleased/query-counts.yml

+---
+title: Track and act upon the number of executed queries
+merge_request:
+author:
+type: added

config/initializers/query_limiting.rb

+if Gitlab::QueryLimiting.enable?
+  require_dependency 'gitlab/query_limiting/active_support_subscriber'
+  require_dependency 'gitlab/query_limiting/transaction'
+  require_dependency 'gitlab/query_limiting/middleware'
+
+  Gitlab::Application.configure do |config|
+    config.middleware.use(Gitlab::QueryLimiting::Middleware)
+  end
+end

doc/development/README.md

@@ -75,6 +75,7 @@ comments: false
 - [Ordering table columns](ordering_table_columns.md)
 - [Verifying database capabilities](verifying_database_capabilities.md)
 - [Database Debugging and Troubleshooting](database_debugging.md)
+- [Query Count Limits](query_count_limits.md)
 ## Testing guides

doc/development/query_count_limits.md

+# Query Count Limits
+
+Each controller or API endpoint is allowed to execute up to 100 SQL queries. In
+a production environment we'll only log an error in case this threshold is
+exceeded, but in a test environment we'll raise an error instead.
+
+## Solving Failing Tests
+
+When a test fails because it executes more than 100 SQL queries there are two
+solutions to this problem:
+
+1. Reduce the number of SQL queries that are executed.
+2. Whitelist the controller or API endpoint.
+
+You should only resort to whitelisting when an existing controller or endpoint
+is to blame as in this case reducing the number of SQL queries can take a lot of
+effort. Newly added controllers and endpoints are not allowed to execute more
+than 100 SQL queries and no exceptions will be made for this rule. _If_ a large
+number of SQL queries is necessary to perform certain work it's best to have
+this work performed by Sidekiq instead of doing this directly in a web request.
+
+## Whitelisting
+
+In the event that you _have_ to whitelist a controller you'll first need to
+create an issue. This issue should (preferably in the title) mention the
+controller or endpoint and include the appropriate labels (`database`,
+`performance`, and at least a team specific label such as `Discussion`).
+
+Once the issue has been created you can whitelist the code in question. For
+Rails controllers it's best to create a `before_action` hook that runs as early
+as possible. The called method in turn should call
+`Gitlab::QueryLimiting.whitelist('issue URL here')`. For example:
+
+```ruby
+class MyController < ApplicationController
+  before_action :whitelist_query_limiting, only: [:show]
+
+  def index
+    # ...
+  end
+
+  def show
+    # ...
+  end
+
+  def whitelist_query_limiting
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/...')
+  end
+end
+```
+
+By using a `before_action` you don't have to modify the controller method in
+question, reducing the likelihood of merge conflicts.
+
+For Grape API endpoints there unfortunately is not a reliable way of running a
+hook before a specific endpoint. This means that you have to add the whitelist
+call directly into the endpoint like so:
+
+```ruby
+get '/projects/:id/foo' do
+  Gitlab::QueryLimiting.whitelist('...')
+
+  # ...
+end
+```

lib/api/branches.rb

@@ -29,6 +29,8 @@ module API
         use :pagination
       end
       get ':id/repository/branches' do
+        Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329')
+
         repository = user_project.repository
         branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name))
         merged_branch_names = repository.merged_branch_names(branches.map(&:name))

lib/api/issues.rb

@@ -161,6 +161,8 @@ module API
         use :issue_params
       end
       post ':id/issues' do
+        Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42320')
+
         authorize! :create_issue, user_project
         # Setting created_at time only allowed for admins and project owners
@@ -201,6 +203,8 @@ module API
                         :labels, :created_at, :due_date, :confidential, :state_event
       end
       put ':id/issues/:issue_iid' do
+        Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322')
+
         issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
         authorize! :update_issue, issue
@@ -234,6 +238,8 @@ module API
         requires :to_project_id, type: Integer, desc: 'The ID of the new project'
       end
       post ':id/issues/:issue_iid/move' do
+        Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323')
+
         issue = user_project.issues.find_by(iid: params[:issue_iid])
         not_found!('Issue') unless issue