Merge branch 'separate-sidekiq-queues' into 'master'

Use separate queues for all Sidekiq workers

## What does this MR do?

This MR updates all workers so that they (mostly) use their own Sidekiq queues. This in turn allows us to monitor queues more accurately and in the future impose queue specific throttles, limits, etc.

This is a critical part we need in 8.13, despite it being so close to release day.

See https://gitlab.com/gitlab-org/gitlab-ce/issues/23370 for more information.

## Does this MR meet the acceptance criteria?

- [x] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG.md) entry added
- [x] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- [x] Conform by the [merge request performance guides](http://docs.gitlab.com/ce/development/merge_request_performance_guidelines.html)
- [x] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [x] Branch has no merge conflicts with `master` (if it does - rebase it please)
- [x] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)

## What are the relevant issue numbers?

https://gitlab.com/gitlab-org/gitlab-ce/issues/23370

See merge request !7006

CHANGELOG.md

@@ -7,6 +7,7 @@ Please view this file on the master branch, on stable branches it's out of date.
 ## 8.13.0 (2016-10-22)
   - Fix save button on project pipeline settings page. (!6955)
+  - All Sidekiq workers now use their own queue
   - Avoid race condition when asynchronously removing expired artifacts. (!6881)
   - Improve Merge When Build Succeeds triggers and execute on pipeline success. (!6675)
   - Respond with 404 Not Found for non-existent tags (Linus Thiel)

app/workers/admin_email_worker.rb

 class AdminEmailWorker
   include Sidekiq::Worker
-
-  sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
+  include CronjobQueue
   def perform
     repository_check_failed_count = Project.where(last_repository_check_failed: true).count

app/workers/build_coverage_worker.rb

 class BuildCoverageWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  include BuildQueue
   def perform(build_id)
     Ci::Build.find_by(id: build_id)

app/workers/build_email_worker.rb

 class BuildEmailWorker
   include Sidekiq::Worker
+  include BuildQueue
   def perform(build_id, recipients, push_data)
     recipients.each do |recipient|

app/workers/build_finished_worker.rb

 class BuildFinishedWorker
   include Sidekiq::Worker
+  include BuildQueue
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|

app/workers/build_hooks_worker.rb

 class BuildHooksWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  include BuildQueue
   def perform(build_id)
     Ci::Build.find_by(id: build_id)

app/workers/build_success_worker.rb

 class BuildSuccessWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  include BuildQueue
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|

app/workers/clear_database_cache_worker.rb

 # This worker clears all cache fields in the database, working in batches.
 class ClearDatabaseCacheWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   BATCH_SIZE = 1000

app/workers/concerns/build_queue.rb

+# Concern for setting Sidekiq settings for the various CI build workers.
+module BuildQueue
+  extend ActiveSupport::Concern
+
+  included do
+    sidekiq_options queue: :build
+  end
+end

app/workers/concerns/cronjob_queue.rb

+# Concern that sets various Sidekiq settings for workers executed using a
+# cronjob.
+module CronjobQueue
+  extend ActiveSupport::Concern
+
+  included do
+    sidekiq_options queue: :cronjob, retry: false
+  end
+end

app/workers/concerns/dedicated_sidekiq_queue.rb

+# Concern that sets the queue of a Sidekiq worker based on the worker's class
+# name/namespace.
+module DedicatedSidekiqQueue
+  extend ActiveSupport::Concern
+
+  included do
+    sidekiq_options queue: name.sub(/Worker\z/, '').underscore.tr('/', '_')
+  end
+end

app/workers/concerns/pipeline_queue.rb

+# Concern for setting Sidekiq settings for the various CI pipeline workers.
+module PipelineQueue
+  extend ActiveSupport::Concern
+
+  included do
+    sidekiq_options queue: :pipeline
+  end
+end

app/workers/concerns/repository_check_queue.rb

+# Concern for setting Sidekiq settings for the various repository check workers.
+module RepositoryCheckQueue
+  extend ActiveSupport::Concern
+
+  included do
+    sidekiq_options queue: :repository_check, retry: false
+  end
+end

app/workers/delete_user_worker.rb

 class DeleteUserWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   def perform(current_user_id, delete_user_id, options = {})
     delete_user  = User.find(delete_user_id)

app/workers/email_receiver_worker.rb

 class EmailReceiverWorker
   include Sidekiq::Worker
-
-  sidekiq_options queue: :incoming_email
+  include DedicatedSidekiqQueue
   def perform(raw)
     return unless Gitlab::IncomingEmail.enabled?

app/workers/emails_on_push_worker.rb

 class EmailsOnPushWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :mailers
+  include DedicatedSidekiqQueue
   attr_reader :email, :skip_premailer
   def perform(project_id, recipients, push_data, options = {})

app/workers/expire_build_artifacts_worker.rb

 class ExpireBuildArtifactsWorker
   include Sidekiq::Worker
+  include CronjobQueue
   def perform
     Rails.logger.info 'Scheduling removal of build artifacts'

app/workers/expire_build_instance_artifacts_worker.rb

 class ExpireBuildInstanceArtifactsWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   def perform(build_id)
     build = Ci::Build

app/workers/git_garbage_collect_worker.rb

 class GitGarbageCollectWorker
   include Sidekiq::Worker
   include Gitlab::ShellAdapter
-  sidekiq_options queue: :gitlab_shell, retry: false
+  include DedicatedSidekiqQueue
+  sidekiq_options retry: false
   def perform(project_id)
     project = Project.find(project_id)

app/workers/gitlab_shell_worker.rb

 class GitlabShellWorker
   include Sidekiq::Worker
   include Gitlab::ShellAdapter
-
-  sidekiq_options queue: :gitlab_shell
+  include DedicatedSidekiqQueue
   def perform(action, *arg)
     gitlab_shell.send(action, *arg)