Use separate queues for all Sidekiq workers

Dumping too many jobs in the same queue (e.g. the "default" queue) is a
dangerous setup. Jobs that take a long time to process can effectively
block any other work from being performed given there are enough of
these jobs.

Furthermore it becomes harder to monitor the jobs as a single queue
could contain jobs for different workers. In such a setup the only
reliable way of getting counts per job is to iterate over all jobs in a
queue, which is a rather time consuming process.

Finally, using separate queues allows us to throttle and prioritize
processing of these queues in the future. For example, the jobs for the
MergeWorker are more important than for example a job used to update
some statistics. Putting all jobs in the same queue would not allow for
this.

This commit includes a Rails migration that moves Sidekiq jobs from the
old queues to the new ones. This migration also takes care of doing the
inverse if ever needed. This does require downtime as otherwise new jobs
could be scheduled in the old queues after this migration completes.

Fixes gitlab-org/gitlab-ce#23370

CHANGELOG.md

@@ -3,6 +3,7 @@ Please view this file on the master branch, on stable branches it's out of date.
 ## 8.13.0 (2016-10-22)
   - Fix save button on project pipeline settings page. (!6955)
+  - All Sidekiq workers now use their own queue
   - Avoid race condition when asynchronously removing expired artifacts. (!6881)
   - Improve Merge When Build Succeeds triggers and execute on pipeline success. (!6675)
   - Respond with 404 Not Found for non-existent tags (Linus Thiel)

app/workers/admin_email_worker.rb

 class AdminEmailWorker
   include Sidekiq::Worker
-  sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
+  sidekiq_options retry: false, queue: :admin_email # this job auto-repeats via sidekiq-cron
   def perform
     repository_check_failed_count = Project.where(last_repository_check_failed: true).count

app/workers/build_coverage_worker.rb

 class BuildCoverageWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :build_coverage
   def perform(build_id)
     Ci::Build.find_by(id: build_id)

app/workers/build_email_worker.rb

 class BuildEmailWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :build_emails
+
   def perform(build_id, recipients, push_data)
     recipients.each do |recipient|
       begin

app/workers/build_finished_worker.rb

 class BuildFinishedWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :finished_builds
+
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
       BuildCoverageWorker.new.perform(build.id)

app/workers/build_hooks_worker.rb

 class BuildHooksWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :build_hooks
   def perform(build_id)
     Ci::Build.find_by(id: build_id)

app/workers/build_success_worker.rb

 class BuildSuccessWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :successful_builds
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|

app/workers/clear_database_cache_worker.rb

 # This worker clears all cache fields in the database, working in batches.
 class ClearDatabaseCacheWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :clear_database_cache
   BATCH_SIZE = 1000

app/workers/delete_user_worker.rb

 class DeleteUserWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :delete_user
   def perform(current_user_id, delete_user_id, options = {})
     delete_user  = User.find(delete_user_id)

app/workers/expire_build_artifacts_worker.rb

 class ExpireBuildArtifactsWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :expire_build_artifacts
+
   def perform
     Rails.logger.info 'Scheduling removal of build artifacts'

app/workers/expire_build_instance_artifacts_worker.rb

 class ExpireBuildInstanceArtifactsWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :expire_build_instance_artifacts
+
   def perform(build_id)
     build = Ci::Build
       .with_expired_artifacts

app/workers/git_garbage_collect_worker.rb

@@ -2,7 +2,7 @@ class GitGarbageCollectWorker
   include Sidekiq::Worker
   include Gitlab::ShellAdapter
-  sidekiq_options queue: :gitlab_shell, retry: false
+  sidekiq_options queue: :git_garbage_collection, retry: false
   def perform(project_id)
     project = Project.find(project_id)

app/workers/group_destroy_worker.rb

 class GroupDestroyWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :delete_group
   def perform(group_id, user_id)
     begin

app/workers/import_export_project_cleanup_worker.rb

 class ImportExportProjectCleanupWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :import_export_project_cleanup
   def perform
     ImportExportCleanUpService.new.execute

app/workers/irker_worker.rb

@@ -4,6 +4,8 @@ require 'socket'
 class IrkerWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :irker
+
   def perform(project_id, chans, colors, push_data, settings)
     project = Project.find(project_id)

app/workers/merge_worker.rb

 class MergeWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :merges
   def perform(merge_request_id, current_user_id, params)
     params = params.with_indifferent_access

app/workers/new_note_worker.rb

 class NewNoteWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :new_note
   def perform(note_id, note_params)
     note = Note.find(note_id)

app/workers/pipeline_hooks_worker.rb

 class PipelineHooksWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :pipeline_hooks
   def perform(pipeline_id)
     Ci::Pipeline.find_by(id: pipeline_id)

app/workers/pipeline_metrics_worker.rb

 class PipelineMetricsWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :pipeline_metrics
   def perform(pipeline_id)
     Ci::Pipeline.find_by(id: pipeline_id).try do |pipeline|

app/workers/pipeline_process_worker.rb

 class PipelineProcessWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :default
+  sidekiq_options queue: :process_pipeline
   def perform(pipeline_id)
     Ci::Pipeline.find_by(id: pipeline_id)