Support multiple Redis instances based on queue type

.flayignore

@@ -3,3 +3,4 @@ lib/gitlab/sanitizers/svg/whitelist.rb
 lib/gitlab/diff/position_tracer.rb
 app/policies/project_policy.rb
 app/models/concerns/relative_positioning.rb
+lib/gitlab/redis/*.rb

.gitignore

@@ -31,6 +31,9 @@ eslint-report.html
 /config/initializers/smtp_settings.rb
 /config/initializers/relative_url.rb
 /config/resque.yml
+/config/redis.cache.yml
+/config/redis.queues.yml
+/config/redis.shared_state.yml
 /config/unicorn.rb
 /config/secrets.yml
 /config/sidekiq.yml

app/controllers/health_controller.rb

@@ -4,7 +4,10 @@ class HealthController < ActionController::Base
   CHECKS = [
     Gitlab::HealthChecks::DbCheck,
-    Gitlab::HealthChecks::RedisCheck,
+    Gitlab::HealthChecks::Redis::RedisCheck,
+    Gitlab::HealthChecks::Redis::CacheCheck,
+    Gitlab::HealthChecks::Redis::QueuesCheck,
+    Gitlab::HealthChecks::Redis::SharedStateCheck,
     Gitlab::HealthChecks::FsShardsCheck
   ].freeze

app/models/ci/runner.rb

@@ -149,7 +149,7 @@ module Ci
     private
     def cleanup_runner_queue
-      Gitlab::Redis.with do |redis|
+      Gitlab::Redis::Queues.with do |redis|
         redis.del(runner_queue_key)
       end
     end

app/models/project.rb

@@ -1386,15 +1386,15 @@ class Project < ActiveRecord::Base
   end
   def pushes_since_gc
-    Gitlab::Redis.with { |redis| redis.get(pushes_since_gc_redis_key).to_i }
+    Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i }
   end
   def increment_pushes_since_gc
-    Gitlab::Redis.with { |redis| redis.incr(pushes_since_gc_redis_key) }
+    Gitlab::Redis::SharedState.with { |redis| redis.incr(pushes_since_gc_redis_shared_state_key) }
   end
   def reset_pushes_since_gc
-    Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) }
+    Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) }
   end
   def route_map_for(commit_sha)
@@ -1457,7 +1457,7 @@ class Project < ActiveRecord::Base
     from && self != from
   end
-  def pushes_since_gc_redis_key
+  def pushes_since_gc_redis_shared_state_key
     "projects/#{id}/pushes_since_gc"
   end

app/services/metrics_service.rb

@@ -3,7 +3,10 @@ require 'prometheus/client/formats/text'
 class MetricsService
   CHECKS = [
     Gitlab::HealthChecks::DbCheck,
-    Gitlab::HealthChecks::RedisCheck,
+    Gitlab::HealthChecks::Redis::RedisCheck,
+    Gitlab::HealthChecks::Redis::CacheCheck,
+    Gitlab::HealthChecks::Redis::QueuesCheck,
+    Gitlab::HealthChecks::Redis::SharedStateCheck,
     Gitlab::HealthChecks::FsShardsCheck
   ].freeze

app/services/milestones/destroy_service.rb

 module Milestones
   class DestroyService < Milestones::BaseService
     def execute(milestone)
-      return unless milestone.is_project_milestone?
-
       Milestone.transaction do
         update_params = { milestone: nil }
         milestone.issues.each do |issue|
-          Issues::UpdateService.new(parent, current_user, update_params).execute(issue)
+          Issues::UpdateService.new(project, current_user, update_params).execute(issue)
         end
         milestone.merge_requests.each do |merge_request|
-          MergeRequests::UpdateService.new(parent, current_user, update_params).execute(merge_request)
+          MergeRequests::UpdateService.new(project, current_user, update_params).execute(merge_request)
         end
         event_service.destroy_milestone(milestone, current_user)

config/README.md

@@ -19,4 +19,132 @@ an ERB file and then loads the resulting YML as its configuration.
 This file is called `resque.yml` for historical reasons. We are **NOT**
 using Resque at the moment. It is used to specify Redis configuration
-values instead.
+values when a single database instance of Redis is desired.
+
+# Advanced Redis configuration files
+
+In more advanced configurations of Redis key-value storage, it is desirable
+to separate the keys by lifecycle and intended use to ease provisioning and
+management of scalable Redis clusters.
+
+These settings provide routing and other configuration data (such as sentinel,
+persistence policies, and other Redis customization) for connections
+to Redis single instances, Redis sentinel, and Redis clusters.
+
+If desired, the routing URL provided by these settings can be used with:
+1. Unix Socket
+    1. named socket for each Redis instance desired.
+    2. `database number` for each Redis instance desired.
+2. TCP Socket
+    1. `host name` or IP for each Redis instance desired
+    2. TCP port number for each Redis instance desired
+    3. `database number` for each Redis instance desired
+    
+## Example URL attribute formats for GitLab Redis `.yml` configuration files
+* Unix Socket, default Redis database (0)
+    * `url: unix:/path/to/redis.sock`
+    * `url: unix:/path/to/redis.sock?db=`
+* Unix Socket, Redis database 44
+    * `url: unix:/path/to/redis.sock?db=44`
+    * `url: unix:/path/to/redis.sock?extra=foo&db=44`
+* TCP Socket for Redis on localhost, port 6379, database 33
+    * `url: redis://:mynewpassword@localhost:6379/33`
+* TCP Socket for Redis on remote host `myserver`, port 6379, database 33
+    * `url: redis://:mynewpassword@myserver:6379/33`
+
+## redis.cache.yml
+
+If configured, `redis.cache.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for `Rails.cache` and other volatile non-persistent data which enhances
+the performance of GitLab.
+Settings here can be overridden by the environment variable
+`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides
+an alternate location for configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `cache` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
+2. URL from `redis.cache.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6380`
+
+The order of precedence for all other configuration settings for `cache`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
+2. the configuration file `redis.cache.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+
+## redis.queues.yml
+
+If configured, `redis.queues.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for clients of `::Gitlab::Redis::Queues`.
+These queues are intended to be the foundation
+of reliable inter-process communication between modules, whether on the same
+host node, or within a cluster.   The primary clients of the queues are
+SideKiq, Mailroom, CI Runner, Workhorse, and push services.  Settings here can
+be overridden by the environment variable
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for
+configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `queues` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
+2. URL from `redis.queues.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6381`
+
+The order of precedence for all other configuration settings for `queues`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
+2. the configuration file `redis.queues.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+
+## redis.shared_state.yml
+
+If configured, `redis.shared_state.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for clients of `::Gitlab::Redis::SharedState` such as session state,
+and rate limiting.
+Settings here can be overridden by the environment variable
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides
+an alternate location for configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `shared_state` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
+2. URL from `redis.shared_state.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6382`
+
+The order of precedence for all other configuration settings for `shared_state`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
+2. the configuration file `redis.shared_state.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+

config/application.rb

@@ -6,7 +6,9 @@ Bundler.require(:default, Rails.env)
 module Gitlab
   class Application < Rails::Application
-    require_dependency Rails.root.join('lib/gitlab/redis')
+    require_dependency Rails.root.join('lib/gitlab/redis/cache')
+    require_dependency Rails.root.join('lib/gitlab/redis/queues')
+    require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
     require_dependency Rails.root.join('lib/gitlab/request_context')
     # Settings in config/environments/* take precedence over those specified here.
@@ -142,15 +144,15 @@ module Gitlab
       end
     end
-    # Use Redis caching across all environments
-    redis_config_hash = Gitlab::Redis.params
-    redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
-    redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
+    # Use caching across all environments
+    caching_config_hash = Gitlab::Redis::Cache.params
+    caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
+    caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
     if Sidekiq.server? # threaded context
-      redis_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
-      redis_config_hash[:pool_timeout] = 1
+      caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
+      caching_config_hash[:pool_timeout] = 1
     end
-    config.cache_store = :redis_store, redis_config_hash
+    config.cache_store = :redis_store, caching_config_hash
     config.active_record.raise_in_transactional_callbacks = true

config/initializers/7_redis.rb

-# Make sure we initialize a Redis connection pool before Sidekiq starts
-# multi-threaded execution.
-Gitlab::Redis.with { nil }
+# Make sure we initialize a Redis connection pool before multi-threaded
+# execution starts by
+# 1. Sidekiq
+# 2. Rails.cache
+# 3. HTTP clients
+Gitlab::Redis::Cache.with { nil }
+Gitlab::Redis::Queues.with { nil }
+Gitlab::Redis::SharedState.with { nil }

config/initializers/peek.rb

-Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.params) }
+Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Cache.params) }
 Peek.into Peek::Views::Host
 Peek.into Peek::Views::PerformanceBar

config/initializers/session_store.rb

@@ -19,12 +19,12 @@ cookie_key = if Rails.env.development?
 if Rails.env.test?
   Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
 else
-  redis_config = Gitlab::Redis.params
-  redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE
+  sessions_config = Gitlab::Redis::SharedState.params
+  sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE
   Gitlab::Application.config.session_store(
     :redis_store, # Using the cookie_store would enable session replay attacks.
-    servers: redis_config,
+    servers: sessions_config,
     key: cookie_key,
     secure: Gitlab.config.gitlab.https,
     httponly: true,

config/initializers/sidekiq.rb

-# Custom Redis configuration
-redis_config_hash = Gitlab::Redis.params
-redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
+# Custom Queues configuration
+queues_config_hash = Gitlab::Redis::Queues.params
+queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE
 # Default is to retry 25 times with exponential backoff. That's too much.
 Sidekiq.default_worker_options = { retry: 3 }
 Sidekiq.configure_server do |config|
-  config.redis = redis_config_hash
+  config.redis = queues_config_hash
   config.server_middleware do |chain|
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
@@ -54,7 +54,7 @@ Sidekiq.configure_server do |config|
 end
 Sidekiq.configure_client do |config|
-  config.redis = redis_config_hash
+  config.redis = queues_config_hash
   config.client_middleware do |chain|
     chain.add Gitlab::SidekiqStatus::ClientMiddleware

config/mail_room.yml

@@ -21,7 +21,7 @@
       :delivery_method: sidekiq
       :delivery_options:
         :redis_url: <%= config[:redis_url].to_json %>
-        :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %>
+        :namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %>
         :queue: email_receiver
         :worker: EmailReceiverWorker
         <% if config[:sentinels] %>
@@ -36,7 +36,7 @@
       :arbitration_method: redis
       :arbitration_options:
         :redis_url: <%= config[:redis_url].to_json %>
-        :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %>
+        :namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %>
         <% if config[:sentinels] %>
         :sentinels:
           <% config[:sentinels].each do |sentinel| %>

config/redis.cache.yml.example

+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+  url: redis://localhost:6379/10
+  #
+  # url: redis://localhost:6380
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26380 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26380 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379/10
+  #
+  # url: redis://localhost:6380
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.cache.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+  ##
+  # url: redis://master:6380
+  # sentinels:
+  #   -
+  #     host: slave1
+  #     port: 26380 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26380 # point to sentinel, not to redis port

config/redis.queues.yml.example

+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+  url: redis://localhost:6379/11
+  #
+  # url: redis://localhost:6381
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26381 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26381 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379/11
+  #
+  # url: redis://localhost:6381
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.queues.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+  ##
+  # url: redis://master:6381
+  # sentinels:
+  #   -
+  #     host: slave1
+  #     port: 26381 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26381 # point to sentinel, not to redis port

config/redis.shared_state.yml.example

+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+  url: redis://localhost:6379/12
+  #
+  # url: redis://localhost:6382
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26382 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26382 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379/12
+  #
+  # url: redis://localhost:6382
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.shared_state.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+  ##
+  # url: redis://master:6382
+  # sentinels:
+  #   -
+  #     host: slave1
+  #     port: 26382 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26382 # point to sentinel, not to redis port

db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb

@@ -56,7 +56,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
   end
   def activities(from, to, page: 1)
-    Gitlab::Redis.with do |redis|
+    Gitlab::Redis::SharedState.with do |redis|
       redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i,
         with_scores: true,
         limit: limit(page))
@@ -64,7 +64,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
   end
   def activities_count(from, to)
-    Gitlab::Redis.with do |redis|
+    Gitlab::Redis::SharedState.with do |redis|
       redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i)
     end
   end

doc/administration/high_availability/redis_source.md

@@ -4,6 +4,11 @@ This is the documentation for configuring a Highly Available Redis setup when
 you have installed Redis all by yourself and not using the bundled one that
 comes with the Omnibus packages.
+Note also that you may elect to override all references to
+`/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis
+settings outlined in
+[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md).
+
 We cannot stress enough the importance of reading the
 [Overview section](redis.md#overview) of the Omnibus Redis HA as it provides
 some invaluable information to the configuration of Redis. Please proceed to

doc/administration/operations/cleaning_up_redis_sessions.md

@@ -15,6 +15,12 @@ prefixed with 'session:gitlab:', so they would look like
 'session:gitlab:976aa289e2189b17d7ef525a6702ace9'. Below we describe how to
 remove the keys in the old format.
+**Note:** the instructions below must be modified in accordance with your
+configuration settings if you have used the advanced Redis
+settings outlined in
+[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md).
+
+
 First we define a shell function with the proper Redis connection details.
 ```