Merge branch '2893-exclude-sidekiq-queues-from-execution-in-sidekiq-cluster' into 'master'

Exclude sidekiq queues from execution in sidekiq-cluster

Closes #2893

See merge request !2571

bin/sidekiq-cluster

 #!/usr/bin/env ruby
 require 'optparse'
+require_relative '../lib/gitlab/sidekiq_config'
 require_relative '../lib/gitlab/sidekiq_cluster'
 require_relative '../lib/gitlab/sidekiq_cluster/cli'

changelogs/unreleased-ee/2893-exclude-sidekiq-queues-from-execution-in-sidekiq-cluster.yml

+---
+title: Allow excluding sidekiq queues from execution in sidekiq-cluster
+merge_request: 2571
+author:

config/initializers/sidekiq.rb

@@ -68,14 +68,12 @@
 # The Sidekiq client API always adds the queue to the Sidekiq queue
 # list, but mail_room and gitlab-shell do not. This is only necessary
 # for monitoring.
-config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s)
+queues = Gitlab::SidekiqConfig.queues
 begin
   Sidekiq.redis do |conn|
     conn.pipelined do
-      config[:queues].each do |queue|
-        conn.sadd('queues', queue[0])
-      end
+      queues.each { |queue| conn.sadd('queues', queue) }
     end
   end
 rescue Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED

doc/administration/operations/extra_sidekiq_processes.md

@@ -87,3 +87,21 @@ command, and not the PID(s) of the started Sidekiq processes.
 The Rails environment can be set by passing the `--environment` flag to the
 `sidekiq-cluster` command, or by setting `RAILS_ENV` to a non-empty value. The
 default value is "development".
+
+## All Queues With Exceptions
+
+You're able to run all queues in `sidekiq_queues.yml` file on a single or 
+multiple processes with exceptions using the `--negate` flag.
+
+For example, say you want to run a single process for all queues, 
+except "process_commit" and "post_receive". You can do so by executing:
+
+```bash
+sidekiq-cluster process_commit,post_receive --negate
+```
+
+For multiple processes of all queues (except "process_commit" and "post_receive"):
+
+```bash
+sidekiq-cluster process_commit,post_receive process_commit,post_receive --negate
+```

lib/gitlab/sidekiq_cluster/cli.rb

@@ -30,7 +30,14 @@ def run(argv = ARGV)
         option_parser.parse!(argv)
-        queues = SidekiqCluster.parse_queues(argv)
+        parsed_queues = SidekiqCluster.parse_queues(argv)
+
+        queues =
+          if @negate_queues
+            parsed_queues.map { |queues| SidekiqConfig.queues(@rails_path, except: queues) }
+          else
+            parsed_queues
+          end
         @logger.info("Starting cluster with #{queues.length} processes")
@@ -93,6 +100,10 @@ def option_parser
             @rails_path = path
           end
+          opt.on('-n', '--negate', 'Run workers for all queues in sidekiq_queues.yml except the given ones') do
+            @negate_queues = true
+          end
+
           opt.on('-i', '--interval INT', 'The number of seconds to wait between worker checks') do |int|
             @interval = int.to_i
           end

lib/gitlab/sidekiq_config.rb

+require 'yaml'
+
+module Gitlab
+  module SidekiqConfig
+    def self.queues(rails_path = Rails.root.to_s, except: [])
+      queues_file_path = File.join(rails_path, 'config', 'sidekiq_queues.yml')
+
+      @queues_file = {}
+      @queues_file[queues_file_path] ||= YAML.load_file(queues_file_path)
+
+      @queues_file[queues_file_path].fetch(:queues).map { |queue, _| queue } - except
+    end
+  end
+end

spec/lib/gitlab/sidekiq_cluster/cli_spec.rb

@@ -19,6 +19,20 @@
         cli.run(%w(foo))
       end
+
+      context 'with --negate flag' do
+        it 'starts Sidekiq workers for all queues on sidekiq_queues.yml except the ones on argv' do
+          expect(Gitlab::SidekiqConfig).to receive(:queues).and_return(['baz'])
+          expect(Gitlab::SidekiqCluster).to receive(:start)
+                                              .with([['baz']], 'test', Dir.pwd)
+                                              .and_return([])
+          expect(cli).to receive(:write_pid)
+          expect(cli).to receive(:trap_signals)
+          expect(cli).to receive(:start_loop)
+
+          cli.run(%w(foo -n))
+        end
+      end
     end
   end

spec/lib/gitlab/sidekiq_config_spec.rb

+require 'rails_helper'
+
+describe Gitlab::SidekiqConfig do
+  describe '.queues' do
+    let(:queues_file_path) { Rails.root.join('config', 'sidekiq_queues.yml') }
+
+    context 'without except argument' do
+      it 'returns all queues defined on config/sidekiq_queues.yml file' do
+        expected_queues = YAML.load_file(queues_file_path)[:queues].map { |queue, _| queue }
+
+        expect(described_class.queues).to eq(expected_queues)
+      end
+    end
+
+    context 'with except argument' do
+      it 'returns queues on config/sidekiq_queues.yml filtering out excluded ones' do
+        expected_queues =
+          YAML.load_file(queues_file_path)[:queues].map { |queue, _| queue } - ['webhook']
+
+        expect(described_class.queues(except: ['webhook'])).to eq(expected_queues)
+      end
+    end
+  end
+end