Gitaly shard connections

CHANGELOG.md

@@ -7,6 +7,7 @@ omnibus-gitlab repository.
 - Remove deprecated satellites configuration
 - Add configuration file for Gitaly
+- Add support for Gitaly address per shard
 9.0.4

files/gitlab-config-template/gitlab.rb.template

@@ -254,7 +254,10 @@ external_url 'GENERATED_EXTERNAL_URL'
 ###! Docs: https://docs.gitlab.com/omnibus/settings/configuration.html#storing-git-data-in-an-alternative-directory
 ###! **If you want to use a single non-default directory to store git data use a
 ###!   path that doesn't contain symlinks.**
-# git_data_dirs({ "default" => { "path" => "/var/opt/gitlab/git-data" } })
+# git_data_dirs({ "default" => { "path" => "/var/opt/gitlab/git-data", 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' } })
+
+### Gitaly settings
+# gitlab_rails['gitaly_enabled'] = true
 ### For storing GitLab application uploads, eg. LFS objects, build artifacts
 ###! Docs: https://docs.gitlab.com/ce/development/shared_files.html

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -139,6 +139,8 @@ default['gitlab']['gitlab-rails']['registry_api_url'] = nil
 default['gitlab']['gitlab-rails']['registry_key_path'] = nil
 default['gitlab']['gitlab-rails']['registry_path'] = nil
 default['gitlab']['gitlab-rails']['registry_issuer'] = "omnibus-gitlab-issuer"
+# Defaults set in libraries/gitlab_rails.rb
+default['gitlab']['gitlab-rails']['repositories_storages'] = {}
 ####
 # These LDAP settings are deprecated in favor of the new syntax. They are kept here for backwards compatibility.
@@ -319,16 +321,14 @@ default['gitlab']['gitlab-shell']['dir'] = "/var/opt/gitlab/gitlab-shell"
 default['gitlab']['gitlab-shell']['log_directory'] = "/var/log/gitlab/gitlab-shell/"
 default['gitlab']['gitlab-shell']['log_level'] = nil
 default['gitlab']['gitlab-shell']['audit_usernames'] = nil
-default['gitlab']['gitlab-shell']['git_data_directories'] = {
-  "default" => { "path" => "/var/opt/gitlab/git-data" }
-}
-default['gitlab']['gitlab-rails']['repositories_storages'] = {
-  "default" => { "path" => "/var/opt/gitlab/git-data/repositories" }
-}
 default['gitlab']['gitlab-shell']['http_settings'] = nil
 default['gitlab']['gitlab-shell']['auth_file'] = nil
 default['gitlab']['gitlab-shell']['git_trace_log_file'] = nil
 default['gitlab']['gitlab-shell']['custom_hooks_dir'] = nil
+# DEPRECATED! Not used by gitlab-shell
+default['gitlab']['gitlab-shell']['git_data_directories'] = {
+  "default" => { "path" => "/var/opt/gitlab/git-data" }
+}
 ###
 # PostgreSQL

files/gitlab-cookbooks/gitlab/libraries/gitaly.rb

@@ -38,5 +38,20 @@ module Gitaly
       Gitlab['gitaly']['env'] = gitaly_env
     end
+
+    def gitaly_address
+      socket_path = user_config['socket_path'] || package_default['socket_path']
+      "unix:#{socket_path}"
+    end
+
+    private
+
+    def user_config
+      Gitlab['gitaly']
+    end
+
+    def package_default
+      Gitlab['node']['gitlab']['gitaly'].to_hash
+    end
   end
 end

files/gitlab-cookbooks/gitlab/libraries/gitlab.rb

@@ -239,6 +239,9 @@ module Gitlab
     def generate_config(node_name)
       generate_secrets(node_name)
       GitlabWorkhorse.parse_variables
+      # Parse Gitaly before gitlab-shell and gitlab-rails
+      # because we need details for repositories_storages
+      Gitaly.parse_variables
       GitlabShell.parse_variables
       GitlabRails.parse_variables
       Logging.parse_variables
@@ -249,7 +252,6 @@ module Gitlab
       GitlabMattermost.parse_variables
       GitlabPages.parse_variables
       Registry.parse_variables
-      Gitaly.parse_variables
       Prometheus.parse_variables
       # Parse nginx variables last because we want all external_url to be
       # parsed first

files/gitlab-cookbooks/gitlab/libraries/gitlab_rails.rb

@@ -16,6 +16,7 @@
 #
 require_relative 'nginx.rb'
+require_relative 'gitaly.rb'
 module GitlabRails
   class << self
@@ -26,6 +27,7 @@ module GitlabRails
       parse_directories
       parse_gitlab_trusted_proxies
       parse_rack_attack_protected_paths
+      parse_gitaly_variables
     end
     def parse_directories
@@ -33,6 +35,7 @@ module GitlabRails
       parse_artifacts_dir
       parse_lfs_objects_dir
       parse_pages_dir
+      parse_repository_storage
     end
     def parse_external_url
@@ -89,6 +92,18 @@ module GitlabRails
       Gitlab['gitlab_rails']['pages_path'] ||= File.join(Gitlab['gitlab_rails']['shared_path'], 'pages')
     end
+    def parse_repository_storage
+      return if Gitlab['gitlab_rails']['repositories_storages']
+      gitaly_address = Gitaly.gitaly_address
+
+      Gitlab['gitlab_rails']['repositories_storages'] ||= {
+        "default" => {
+          "path" => "/var/opt/gitlab/git-data/repositories",
+          "gitaly_address" => gitaly_address
+        }
+      }
+    end
+
     def parse_gitlab_trusted_proxies
       Gitlab['nginx']['real_ip_trusted_addresses'] ||= Gitlab['node']['gitlab']['nginx']['real_ip_trusted_addresses']
       Gitlab['gitlab_rails']['trusted_proxies'] ||= Gitlab['nginx']['real_ip_trusted_addresses']
@@ -136,6 +151,15 @@ module GitlabRails
       "#{Gitlab['node']['package']['install-dir']}/embedded/service/gitlab-rails/public"
     end
+    def parse_gitaly_variables
+      return unless Gitlab['gitlab_rails']['gitaly_enabled'].nil?
+
+      gitaly_enabled = Gitlab['gitaly']['enable']
+      gitaly_enabled = Gitlab['node']['gitlab']['gitaly']['enable'] if gitaly_enabled.nil?
+
+      Gitlab['gitlab_rails']['gitaly_enabled'] = gitaly_enabled
+    end
+
     private
     def any_service_role_defined?

files/gitlab-cookbooks/gitlab/libraries/gitlab_shell.rb

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+require_relative 'gitaly.rb'
 module GitlabShell
   class << self
@@ -26,6 +27,7 @@ module GitlabShell
       git_data_dirs = Gitlab['git_data_dirs']
       git_data_dir = Gitlab['git_data_dir']
       return unless git_data_dirs.any? || git_data_dir
+      gitaly_address = Gitaly.gitaly_address
       Gitlab['gitlab_shell']['git_data_directories'] ||=
         if git_data_dirs.any?
@@ -43,7 +45,7 @@ module GitlabShell
       Gitlab['gitlab_rails']['repositories_storages'] ||=
         Hash[Gitlab['gitlab_shell']['git_data_directories'].map do |name, data_directory|
-          [name, { 'path' => File.join(data_directory['path'], 'repositories') }]
+          [name, { 'path' => File.join(data_directory['path'], 'repositories'), 'gitaly_address' => gitaly_address }]
         end]
     end

files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb

@@ -227,8 +227,6 @@ templatesymlink "Create a relative_url.rb and create a symlink to Rails root" do
   end
 end
-gitaly_socket =  node['gitlab']['gitaly']['socket_path'] if node['gitlab']['gitaly']['enable']
-
 templatesymlink "Create a gitlab.yml and create a symlink to Rails root" do
   link_from File.join(gitlab_rails_source_dir, "config/gitlab.yml")
   link_to File.join(gitlab_rails_etc_dir, "gitlab.yml")
@@ -247,8 +245,7 @@ templatesymlink "Create a gitlab.yml and create a symlink to Rails root" do
       pages_external_http: node['gitlab']['gitlab-pages']['external_http'],
       pages_external_https: node['gitlab']['gitlab-pages']['external_https'],
       mattermost_host: mattermost_host,
-      mattermost_enabled: node['gitlab']['mattermost']['enable'] || !mattermost_host.nil?,
-      gitaly_socket: gitaly_socket
+      mattermost_enabled: node['gitlab']['mattermost']['enable'] || !mattermost_host.nil?
     )
   )
   restarts dependent_services

files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb

@@ -349,18 +349,13 @@ production: &base
   shared:
     path: <%= @shared_path %>
-<% if @gitaly_socket %>
   # Gitaly settings
+  # This setting controls whether GitLab uses Gitaly
+  # Eventually Gitaly use will become mandatory and
+  # this option will disappear.
   gitaly:
-    socket_path: <%= @gitaly_socket %>
-    # The socket_path setting is optional and obsolete. When this is set
-    # GitLab assumes it can reach a Gitaly services via a Unix socket at
-    # this path. When this is commented out GitLab will not use Gitaly.
-    #
-    # This setting is obsolete because we expect it to be moved under
-    # repositories/storages in GitLab 9.1.
-    #
-<% end %>
+    enabled: <%= @gitaly_enabled %>
+
   #
   # 4. Advanced settings

spec/chef/gitlab_rails_spec.rb

-require 'chef_helper'
-require 'base64'
-
-describe 'gitlab_rails' do
-  let(:chef_run) { ChefSpec::SoloRunner.new.converge('gitlab::default') }
-
-  context 'when there is a legacy GitLab Rails stuck_ci_builds_worker_cron key' do
-    before do
-      allow(Gitlab).to receive(:[]).and_call_original
-      stub_gitlab_rb(gitlab_rails: { stuck_ci_builds_worker_cron: '0 1 2 * *' })
-    end
-
-    it 'warns that this value is deprecated' do
-      allow(Chef::Log).to receive(:warn).and_call_original
-      expect(Chef::Log).to receive(:warn).with(/gitlab_rails\['stuck_ci_builds_worker_cron'\]/)
-
-      chef_run
-    end
-
-    it 'copies legacy value from legacy key to new one' do
-      chef_run
-
-      expect(Gitlab['gitlab_rails']['stuck_ci_jobs_worker_cron']).to eq('0 1 2 * *')
-    end
-  end
-end

spec/chef/recipes/gitlab-rails_spec.rb

@@ -201,27 +201,66 @@ describe 'gitlab::gitlab-rails' do
     context 'Gitaly settings' do
       context 'by default' do
-        it 'sets the path to socket' do
+        it 'is enabled' do
           expect(chef_run).to render_file(gitlab_yml_path)
-            .with_content(%r{gitaly:\s+socket_path:\s+/var/opt/gitlab/gitaly/gitaly.socket})
+            .with_content(%r{gitaly:\s+enabled: true})
         end
-        context 'when socket path is changed' do
-          it 'sets the path to socket' do
-            stub_gitlab_rb(gitaly: { socket_path: '/tmp/socket' })
+      end
+
+      context 'when gitaly is disabled' do
+        it 'disables gitaly in config' do
+          stub_gitlab_rb(gitaly: { enable: false })
+
+          expect(chef_run).to render_file(gitlab_yml_path)
+            .with_content(%r{gitaly:\s+enabled: false})
+        end
+      end
+
+      context 'when gitaly service is connecting from a different node' do
+        before do
+          stub_gitlab_rb(
+            gitaly: { enable: false },
+            gitlab_rails: { gitaly_enabled: true }
+          )
+        end
+
+        it 'enables gitaly in config' do
+          expect(chef_run).to render_file(gitlab_yml_path)
+            .with_content(%r{gitaly:\s+enabled: true})
+        end
+      end
+
+      context 'when gitaly service is running on a different node' do
+        before do
+          stub_gitlab_rb(
+            gitlab_rails: { gitaly_enabled: false }
+          )
+          it 'disables gitaly in config' do
             expect(chef_run).to render_file(gitlab_yml_path)
-              .with_content(%r{gitaly:\s+socket_path:\s+/tmp/socket})
+              .with_content(%r{gitaly:\s+enabled: false})
           end
         end
       end
-      context 'when gitaly is disabled' do
-        it 'sets the mattermost host' do
-          stub_gitlab_rb(gitaly: { enable: false })
-          expect(chef_run).not_to render_file(gitlab_yml_path)
-            .with_content(%r{gitaly:\s+socket_path:\s+/var/opt/gitlab/gitaly/gitaly.socket})
-        end
+    end
+    context 'when there is a legacy GitLab Rails stuck_ci_builds_worker_cron key' do
+      before do
+        allow(Gitlab).to receive(:[]).and_call_original
+        stub_gitlab_rb(gitlab_rails: { stuck_ci_builds_worker_cron: '0 1 2 * *' })
+      end
+      it 'warns that this value is deprecated' do
+        allow(Chef::Log).to receive(:warn).and_call_original
+        expect(Chef::Log).to receive(:warn).with(/gitlab_rails\['stuck_ci_builds_worker_cron'\]/)
+
+        chef_run
+      end
+
+      it 'copies legacy value from legacy key to new one' do
+        chef_run
+
+        expect(Gitlab['gitlab_rails']['stuck_ci_jobs_worker_cron']).to eq('0 1 2 * *')
       end
     end
   end

spec/chef/recipes/gitlab-shell_spec.rb

@@ -236,7 +236,7 @@ describe 'gitlab_shell::git_data_dir' do
     it 'correctly sets the repository storage directories' do
       expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages'])
-        .to eql('default' => { 'path' => '/tmp/user/git-data/repositories' })
+        .to eql('default' => { 'path' => '/tmp/user/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' })
     end
   end
@@ -259,8 +259,8 @@ describe 'gitlab_shell::git_data_dir' do
     it 'correctly sets the repository storage directories' do
       expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
-                                                                                        'default' => { 'path' => '/tmp/default/git-data/repositories' },
-                                                                                        'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories' }
+                                                                                        'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
+                                                                                        'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' }
                                                                                       })
     end
   end
@@ -287,8 +287,8 @@ describe 'gitlab_shell::git_data_dir' do
     it 'correctly sets the repository storage directories' do
       expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
-                                                                                        'default' => { 'path' => '/tmp/default/git-data/repositories' },
-                                                                                        'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories' }
+                                                                                        'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
+                                                                                        'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' }
                                                                                       })
     end
   end