Use separate queues for all Sidekiq workers

Dumping too many jobs in the same queue (e.g. the "default" queue) is a
dangerous setup. Jobs that take a long time to process can effectively
block any other work from being performed given there are enough of
these jobs.

Furthermore it becomes harder to monitor the jobs as a single queue
could contain jobs for different workers. In such a setup the only
reliable way of getting counts per job is to iterate over all jobs in a
queue, which is a rather time consuming process.

Finally, using separate queues allows us to throttle and prioritize
processing of these queues in the future. For example, the jobs for the
MergeWorker are more important than for example a job used to update
some statistics. Putting all jobs in the same queue would not allow for
this.

This commit includes a Rails migration that moves Sidekiq jobs from the
old queues to the new ones. This migration also takes care of doing the
inverse if ever needed. This does require downtime as otherwise new jobs
could be scheduled in the old queues after this migration completes.

Fixes gitlab-org/gitlab-ce#23370

CHANGELOG.md

@@ -7,6 +7,7 @@ Please view this file on the master branch, on stable branches it's out of date.
 ## 8.13.0 (2016-10-22)
   - Fix save button on project pipeline settings page. (!6955)
+  - All Sidekiq workers now use their own queue
   - Avoid race condition when asynchronously removing expired artifacts. (!6881)
   - Improve Merge When Build Succeeds triggers and execute on pipeline success. (!6675)
   - Respond with 404 Not Found for non-existent tags (Linus Thiel)

app/workers/admin_email_worker.rb

 class AdminEmailWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   sidekiq_options retry: false # this job auto-repeats via sidekiq-cron

app/workers/build_coverage_worker.rb

 class BuildCoverageWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  include DedicatedSidekiqQueue
   def perform(build_id)
     Ci::Build.find_by(id: build_id)

app/workers/build_email_worker.rb

 class BuildEmailWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   def perform(build_id, recipients, push_data)
     recipients.each do |recipient|

app/workers/build_finished_worker.rb

 class BuildFinishedWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|

app/workers/build_hooks_worker.rb

 class BuildHooksWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  include DedicatedSidekiqQueue
   def perform(build_id)
     Ci::Build.find_by(id: build_id)

app/workers/build_success_worker.rb

 class BuildSuccessWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  include DedicatedSidekiqQueue
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|

app/workers/clear_database_cache_worker.rb

 # This worker clears all cache fields in the database, working in batches.
 class ClearDatabaseCacheWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   BATCH_SIZE = 1000

app/workers/concerns/dedicated_sidekiq_queue.rb

+module DedicatedSidekiqQueue
+  extend ActiveSupport::Concern
+
+  included do
+    sidekiq_options queue: name.sub(/Worker\z/, '').underscore.tr('/', '_')
+  end
+end

app/workers/delete_user_worker.rb

 class DeleteUserWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   def perform(current_user_id, delete_user_id, options = {})
     delete_user  = User.find(delete_user_id)

app/workers/email_receiver_worker.rb

 class EmailReceiverWorker
   include Sidekiq::Worker
-
-  sidekiq_options queue: :incoming_email
+  include DedicatedSidekiqQueue
   def perform(raw)
     return unless Gitlab::IncomingEmail.enabled?

app/workers/emails_on_push_worker.rb

 class EmailsOnPushWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :mailers
+  include DedicatedSidekiqQueue
   attr_reader :email, :skip_premailer
   def perform(project_id, recipients, push_data, options = {})

app/workers/expire_build_artifacts_worker.rb

 class ExpireBuildArtifactsWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   def perform
     Rails.logger.info 'Scheduling removal of build artifacts'

app/workers/expire_build_instance_artifacts_worker.rb

 class ExpireBuildInstanceArtifactsWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   def perform(build_id)
     build = Ci::Build

app/workers/git_garbage_collect_worker.rb

 class GitGarbageCollectWorker
   include Sidekiq::Worker
   include Gitlab::ShellAdapter
-  sidekiq_options queue: :gitlab_shell, retry: false
+  include DedicatedSidekiqQueue
+  sidekiq_options retry: false
   def perform(project_id)
     project = Project.find(project_id)

app/workers/gitlab_shell_worker.rb

 class GitlabShellWorker
   include Sidekiq::Worker
   include Gitlab::ShellAdapter
-
-  sidekiq_options queue: :gitlab_shell
+  include DedicatedSidekiqQueue
   def perform(action, *arg)
     gitlab_shell.send(action, *arg)

app/workers/group_destroy_worker.rb

 class GroupDestroyWorker
   include Sidekiq::Worker
-
-  sidekiq_options queue: :default
+  include DedicatedSidekiqQueue
   def perform(group_id, user_id)
     begin

app/workers/import_export_project_cleanup_worker.rb

 class ImportExportProjectCleanupWorker
   include Sidekiq::Worker
-
-  sidekiq_options queue: :default
+  include DedicatedSidekiqQueue
   def perform
     ImportExportCleanUpService.new.execute

app/workers/irker_worker.rb

@@ -3,6 +3,7 @@ require 'socket'
 class IrkerWorker
   include Sidekiq::Worker
+  include DedicatedSidekiqQueue
   def perform(project_id, chans, colors, push_data, settings)
     project = Project.find(project_id)

app/workers/merge_worker.rb

 class MergeWorker
   include Sidekiq::Worker
-
-  sidekiq_options queue: :default
+  include DedicatedSidekiqQueue
   def perform(merge_request_id, current_user_id, params)
     params = params.with_indifferent_access