Allow certain Sidekiq jobs to be throttled

Gemfile

@@ -137,6 +137,7 @@ gem 'acts-as-taggable-on', '~> 4.0'
 gem 'sidekiq', '~> 4.2'
 gem 'sidekiq-cron', '~> 0.4.0'
 gem 'redis-namespace', '~> 1.5.2'
+gem 'sidekiq-limit_fetch', '~> 3.4'
 # HTTP requests
 gem 'httparty', '~> 0.13.3'

Gemfile.lock

@@ -685,6 +685,8 @@ GEM
       redis-namespace (>= 1.5.2)
       rufus-scheduler (>= 2.0.24)
       sidekiq (>= 4.0.0)
+    sidekiq-limit_fetch (3.4.0)
+      sidekiq (>= 4)
     simplecov (0.12.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
@@ -961,6 +963,7 @@ DEPENDENCIES
   shoulda-matchers (~> 2.8.0)
   sidekiq (~> 4.2)
   sidekiq-cron (~> 0.4.0)
+  sidekiq-limit_fetch (~> 3.4)
   simplecov (= 0.12.0)
   slack-notifier (~> 1.2.0)
   spinach-rails (~> 0.2.1)

app/controllers/admin/application_settings_controller.rb

@@ -117,6 +117,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :send_user_confirmation_email,
       :container_registry_token_expire_delay,
       :enabled_git_access_protocol,
+      :sidekiq_throttling_enabled,
       :housekeeping_enabled,
       :housekeeping_bitmaps_enabled,
       :housekeeping_incremental_repack_period,

app/models/application_setting.rb

@@ -180,6 +180,7 @@ class ApplicationSetting < ActiveRecord::Base
       container_registry_token_expire_delay: 5,
       repository_storages: ['default'],
       user_default_external: false,
+      sidekiq_throttling_enabled: false,
       housekeeping_enabled: true,
       housekeeping_bitmaps_enabled: true,
       housekeeping_incremental_repack_period: 10,

app/views/admin/application_settings/_form.html.haml

@@ -283,6 +283,19 @@
           The amount of points to store in a single UDP packet. More points
           results in fewer but larger UDP packets being sent.
+  %fieldset
+    %legend Background Jobs
+    %p
+      These settings require a restart to take effect.
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :sidekiq_throttling_enabled do
+            = f.check_box :sidekiq_throttling_enabled
+            Enable Sidekiq Job Throttling
+          .help-block
+            Limit the amount of resources slow running jobs are assigned.
+
   %fieldset
     %legend Spam and Anti-bot Protection
     .form-group

config/initializers/sidekiq.rb

+require 'gitlab/current_settings'
+include Gitlab::CurrentSettings
+
 # Custom Redis configuration
 redis_config_hash = Gitlab::Redis.params
 redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
@@ -29,6 +32,19 @@ Sidekiq.configure_server do |config|
   end
   Sidekiq::Cron::Job.load_from_hash! cron_jobs
+  # allow it to fail: it may do so when create_from_defaults is executed before migrations are actually done
+  begin
+    throttling_enabled = current_application_settings.sidekiq_throttling_enabled
+  rescue
+    throttling_enabled = false
+  end
+
+  if throttling_enabled
+    { 'project_cache' => 0.1, 'pipeline' => 0.1 }.each do |queue, ratio|
+      Sidekiq::Queue[queue].limit = (ratio * Sidekiq.options[:concurrency]).ceil
+    end
+  end
+
   # Database pool should be at least `sidekiq_concurrency` + 2
   # For more info, see: https://github.com/mperham/sidekiq/blob/master/4.0-Upgrade.md
   config = ActiveRecord::Base.configurations[Rails.env] ||

db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddSidekiqThrottlingToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  # DOWNTIME_REASON = ''
+
+  # When using the methods "add_concurrent_index" or "add_column_with_default"
+  # you must disable the use of transactions as these methods can not run in an
+  # existing transaction. When using "add_concurrent_index" make sure that this
+  # method is the _only_ method called in the migration, any other changes
+  # should go in a separate migration. This ensures that upon failure _only_ the
+  # index creation fails and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+  # disable_ddl_transaction!
+
+  def change
+    add_column :application_settings, :sidekiq_throttling_enabled, :boolean, default: false
+  end
+end

db/schema.rb

@@ -98,6 +98,7 @@ ActiveRecord::Schema.define(version: 20161106185620) do
     t.text "help_page_text_html"
     t.text "shared_runners_text_html"
     t.text "after_sign_up_text_html"
+    t.boolean "sidekiq_throttling_enabled", default: false
     t.boolean "housekeeping_enabled", default: true, null: false
     t.boolean "housekeeping_bitmaps_enabled", default: true, null: false
     t.integer "housekeeping_incremental_repack_period", default: 10, null: false

lib/gitlab/current_settings.rb

@@ -50,6 +50,7 @@ module Gitlab
         repository_checks_enabled: true,
         container_registry_token_expire_delay: 5,
         user_default_external: false,
+        sidekiq_throttling_enabled: false,
       )
     end