Merge remote-tracking branch 'origin/master' into pitr

changelogs/unreleased/4589.yml

+---
+title: Explicitly set group for repositories_storages and improve manage-storage-directories tests
+merge_request: 4589
+author: Ben Bodenmiller (@bbodenmiller)
+type: other

changelogs/unreleased/4594.yml

+---
+title: Remove EE only note for packages in gitlab.yml.erb
+merge_request: 4594
+author: Ben Bodenmiller (@bbodenmiller)
+type: fixed

changelogs/unreleased/5672-consolidated-form-for-object-storage-does-not-work-in-docker-install-of-gitlab.yml

+---
+title: Fix onsolidated form object storage configuration template
+merge_request: 4656
+author:
+type: fixed

changelogs/unreleased/brodock-patroni-standbycluster.yml

+---
+title: Allow bootstrapping patroni Standby Cluster
+merge_request: 4558
+author:
+type: added

changelogs/unreleased/john_long-add-selinux-module-for-gitlab-shell.yml

+---
+title: Add selinux module for gitlab-shell
+merge_request: 4598
+author:
+type: fixed

changelogs/unreleased/postgresql-cert-hba.yml

+---
+title: Add support for cert authentication with PostgreSQL
+merge_request: 4618
+author:
+type: changed

doc/settings/configuration.md

@@ -513,7 +513,7 @@ Enabling this setting will prevent the creation of the following directories:
 | Default location                                       | Permissions | Ownership        | Purpose |
 |--------------------------------------------------------|-------------|------------------|---------|
 | `/var/opt/gitlab/git-data`                             | `0700`        | `git`            | Holds repositories directory |
-| `/var/opt/gitlab/git-data/repositories`                | `2770`        | `git`            | Holds Git repositories |
+| `/var/opt/gitlab/git-data/repositories`                | `2770`        | `git:git`        | Holds Git repositories |
 | `/var/opt/gitlab/gitlab-rails/shared`                  | `0751`        | `git:gitlab-www` | Holds large object directories |
 | `/var/opt/gitlab/gitlab-rails/shared/artifacts`        | `0700`        | `git`            | Holds CI artifacts |
 | `/var/opt/gitlab/gitlab-rails/shared/external-diffs`   | `0700`        | `git`            | Holds external merge request diffs |
@@ -521,8 +521,8 @@ Enabling this setting will prevent the creation of the following directories:
 | `/var/opt/gitlab/gitlab-rails/shared/packages`         | `0700`        | `git`            | Holds package repository |
 | `/var/opt/gitlab/gitlab-rails/shared/dependency_proxy` | `0700`        | `git`            | Holds dependency proxy |
 | `/var/opt/gitlab/gitlab-rails/shared/terraform_state`  | `0700`        | `git`            | Holds terraform state |
-| `/var/opt/gitlab/gitlab-rails/uploads`                 | `0700`        | `git`            | Holds user attachments |
 | `/var/opt/gitlab/gitlab-rails/shared/pages`            | `0750`        | `git:gitlab-www` | Holds user pages |
+| `/var/opt/gitlab/gitlab-rails/uploads`                 | `0700`        | `git`            | Holds user attachments |
 | `/var/opt/gitlab/gitlab-ci/builds`                     | `0700`        | `git`            | Holds CI build logs |
 | `/var/opt/gitlab/.ssh`                                 | `0700`        | `git:git`        | Holds authorized keys |

doc/settings/database.md

@@ -168,7 +168,59 @@ gitlabhq_production=> SELECT * FROM pg_stat_ssl;
 (19 rows)
 ```
-Rows that have `t` listed under the `ssl` column are enabled.
+1. Rows that have `t` listed under the `ssl` column are enabled.
+1. Rows that have a value in the `clientdn` are using the `cert` authentication method
+
+#### Configure SSL client authentication
+
+Client SSL certificates can be used to authenticate to the database server. Creating the certificates
+is beyond the scope of `omnibus-gitlab`. But users who have an existing SSL certificate management solution
+can use this.
+
+##### Configure the database server
+
+1. Create a certificate and key for the server, the common name should equal the DNS name of the server
+1. Copy the server certificate, key, and CA file to the PostgreSQL server, and ensure the permissions are correct
+   1. The certificate should be owned by the database user (default: `gitlab-psql`)
+   1. The key file should be owned by the database user, and its permissions should be `0400`
+   1. The CA file should be owned by the database user, and its permissions should be `0400`
+
+   NOTE: **Note:**
+   Do not use the filenames `server.crt` or `server.key` for these files. These are reserved for internal use of `omnibus-gitlab`.
+
+1. Ensure the following is set in `gitlab.rb`.
+
+   ```ruby
+   postgresql['ssl_cert_file'] = 'PATH_TO_CERTIFICATE'
+   postgresql['ssl_key_file'] = 'PATH_TO_KEY_FILE'
+   postgresql['ssl_ca_file'] = 'PATH_TO_CA_FILE'
+   postgresql['listen_address'] = 'IP_ADDRESS'
+   postgresql['cert_auth_addresses'] = {
+   'IP_ADDRESS' => {
+     'database' => 'gitlabhq_production',
+     'user' => 'gitlab'
+   }
+   ```
+
+   `listen_address` should be set to an IP address of the server that the clients will use to connect to the database.
+   `cert_auth_addresses` should contain a list of IP addresses, and the databases and users that are allowed to connect to the database.
+1. Run `gitlab-ctl reconfigure` then `gitlab-ctl restart postgresql` in order for the new settings to take effect
+
+#### Configure the Rails client
+
+In order for the rails client to connect to the server, you will need a certficate and key with the `commonName` set to `gitlab`, that is signed by a certificate authority trusted in the CA file specified in `ssl_ca_file` on the database server.
+
+1. Configure `gitlab.rb`
+
+   ```ruby
+   gitlab_rails['db_host'] = 'IP_ADDRESS_OR_HOSTNAME_OF_DATABASE_SERVER'
+   gitlab_rails['db_sslcert'] = 'PATH_TO_CERTIFICATE_FILE'
+   gitlab_rails['db_sslkey'] = 'PATH_TO_KEY_FILE'
+   gitlab_rails['db_rootcert'] = 'PATH_TO_CA_FILE'
+   ```
+
+1. Run `gitlab-ctl reconfigure` for the rails client to use the new settings
+1. Follow the steps in [Verifying that SSL is being used](#verifying-that-ssl-is-being-used) to ensure the authentication is working.
 ### Configure packaged PostgreSQL server to listen on TCP/IP

files/gitlab-config-template/gitlab.rb.template

@@ -278,18 +278,17 @@ external_url 'GENERATED_EXTERNAL_URL'
 ###!
 ###! Be sure to use different buckets for each type of object.
 ###! Docs: https://docs.gitlab.com/ee/administration/object_storage.html
-gitlab_rails['object_store']['enabled'] = false
-gitlab_rails['object_store']['connection'] = {}
-gitlab_rails['object_store']['storage_options'] = {}
-gitlab_rails['object_store']['proxy_download'] = false
-gitlab_rails['object_store']['objects']['artifacts']['bucket'] = nil
-gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = nil
-gitlab_rails['object_store']['objects']['lfs']['bucket'] = nil
-gitlab_rails['object_store']['objects']['uploads']['bucket'] = nil
-gitlab_rails['object_store']['objects']['packages']['bucket'] = nil
-gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = nil
-gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = nil
-gitlab_rails['object_store']['objects']['pages']['bucket'] = nil
+# gitlab_rails['object_store']['enabled'] = false
+# gitlab_rails['object_store']['connection'] = {}
+# gitlab_rails['object_store']['storage_options'] = {}
+# gitlab_rails['object_store']['proxy_download'] = false
+# gitlab_rails['object_store']['objects']['artifacts']['bucket'] = nil
+# gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = nil
+# gitlab_rails['object_store']['objects']['lfs']['bucket'] = nil
+# gitlab_rails['object_store']['objects']['uploads']['bucket'] = nil
+# gitlab_rails['object_store']['objects']['packages']['bucket'] = nil
+# gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = nil
+# gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = nil
 ### Job Artifacts
 # gitlab_rails['artifacts_enabled'] = true
@@ -800,7 +799,7 @@ gitlab_rails['object_store']['objects']['pages']['bucket'] = nil
 # gitlab_workhorse['ha'] = false
 # gitlab_workhorse['listen_network'] = "unix"
 # gitlab_workhorse['listen_umask'] = 000
-# gitlab_workhorse['listen_addr'] = "/var/opt/gitlab/gitlab-workhorse/socket"
+# gitlab_workhorse['listen_addr'] = "/var/opt/gitlab/gitlab-workhorse/sockets/socket"
 # gitlab_workhorse['auth_backend'] = "http://localhost:8080"
 ##! the empty string is the default in gitlab-workhorse option parser
@@ -1048,6 +1047,12 @@ gitlab_rails['object_store']['objects']['pages']['bucket'] = nil
 # postgresql['ssl_key_file'] = 'server.key'
 # postgresql['ssl_ca_file'] = '/opt/gitlab/embedded/ssl/certs/cacert.pem'
 # postgresql['ssl_crl_file'] = nil
+# postgresql['cert_auth_addresses'] = {
+#   'ADDRESS' => {
+#     database: 'gitlabhq_production',
+#     user: 'gitlab'
+#   }
+# }
 ### Replication settings
 ###! Note, some replication settings do not require a full restart. They are documented below.

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -615,7 +615,8 @@ default['gitlab']['gitlab-workhorse']['enable'] = false
 default['gitlab']['gitlab-workhorse']['ha'] = false
 default['gitlab']['gitlab-workhorse']['listen_network'] = "unix"
 default['gitlab']['gitlab-workhorse']['listen_umask'] = 000
-default['gitlab']['gitlab-workhorse']['listen_addr'] = "/var/opt/gitlab/gitlab-workhorse/socket"
+default['gitlab']['gitlab-workhorse']['sockets_directory'] = nil
+default['gitlab']['gitlab-workhorse']['listen_addr'] = nil
 default['gitlab']['gitlab-workhorse']['auth_backend'] = "http://localhost:8080"
 default['gitlab']['gitlab-workhorse']['auth_socket'] = nil
 default['gitlab']['gitlab-workhorse']['cable_backend'] = "http://localhost:8280"

files/gitlab-cookbooks/gitlab/libraries/gitlab_workhorse.rb

@@ -19,6 +19,17 @@ module GitlabWorkhorse
   class << self
     def parse_variables
       Gitlab['gitlab_workhorse']['auth_socket'] = nil if !auth_socket_specified? && auth_backend_specified?
+
+      user_listen_addr = Gitlab['gitlab_workhorse']['listen_addr']
+      Gitlab['gitlab_workhorse']['sockets_directory'] ||= '/var/opt/gitlab/gitlab-workhorse/sockets' if user_listen_addr.nil?
+
+      sockets_dir = Gitlab['gitlab_workhorse']['sockets_directory']
+
+      default_network = Gitlab['node']['gitlab']['gitlab-workhorse']['listen_network']
+      user_network = Gitlab['gitlab_workhorse']['listen_network']
+      network = user_network || default_network
+
+      Gitlab['gitlab_workhorse']['listen_addr'] ||= File.join(sockets_dir, 'socket') if network == "unix"
     end
     def parse_secrets

files/gitlab-cookbooks/gitlab/libraries/helpers/gitlab_workhorse_helper.rb

+require_relative 'base_helper'
+
+class GitlabWorkhorseHelper < BaseHelper
+  attr_reader :node
+
+  def unix_socket?
+    node['gitlab']['gitlab-workhorse']['listen_network'] == "unix"
+  end
+end

files/gitlab-cookbooks/gitlab/libraries/helpers/web_server_helper.rb

@@ -14,14 +14,15 @@ class WebServerHelper
     end
     def internal_api_url(node)
+      workhorse_helper = GitlabWorkhorseHelper.new(node)
+
       gitlab_url = node['gitlab']['gitlab-rails']['internal_api_url']
       # If no internal_api_url is specified, default to Workhorse settings
-      use_socket = node['gitlab']['gitlab-workhorse']['listen_network'] == "unix"
       workhorse_url = node['gitlab']['gitlab-workhorse']['listen_addr']
       relative_path = Gitlab['gitlab_workhorse']['relative_url']
-      gitlab_url ||= use_socket ? "http+unix://#{ERB::Util.url_encode(workhorse_url)}" : "http://#{workhorse_url}#{relative_path}"
-      gitlab_relative_path = relative_path || '' if use_socket
+      gitlab_url ||= workhorse_helper.unix_socket? ? "http+unix://#{ERB::Util.url_encode(workhorse_url)}" : "http://#{workhorse_url}#{relative_path}"
+      gitlab_relative_path = relative_path || '' if workhorse_helper.unix_socket?
       [gitlab_url, gitlab_relative_path]
     end

files/gitlab-cookbooks/gitlab/recipes/gitlab-healthcheck.rb

@@ -15,6 +15,8 @@
 # limitations under the License.
 #
+workhorse_helper = GitlabWorkhorseHelper.new(node)
+
 # If nginx is disabled we will use workhorse for the healthcheck
 if node['gitlab']['nginx']['enable']
   listen_https = node['gitlab']['nginx']['listen_https']
@@ -26,7 +28,7 @@ if node['gitlab']['nginx']['enable']
 else
   # Always use http for workhorse
   schema = 'http'
-  use_socket = node['gitlab']['gitlab-workhorse']['listen_network'] == "unix"
+  use_socket = workhorse_helper.unix_socket?
   host = use_socket ? 'localhost' : node['gitlab']['gitlab-workhorse']['listen_addr']
 end

files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb

@@ -56,6 +56,7 @@ repositories_storages = node['gitlab']['gitlab-rails']['repositories_storages']
 repositories_storages.each do |_name, repositories_storage|
   storage_directory repositories_storage['path'] do
     owner gitlab_user
+    group gitlab_group
     mode "2770"
   end
 end
@@ -342,6 +343,7 @@ templatesymlink "Create a gitlab_shell_secret and create a symlink to Rails root
   sensitive true
   variables(secret_token: node['gitlab']['gitlab-shell']['secret_token'])
   dependent_services.each { |svc| notifies :restart, svc }
+  notifies :run, 'bash[Set proper security context on ssh files for selinux]', :delayed if SELinuxHelper.enabled?
 end
 gitlab_pages_services = dependent_services

files/gitlab-cookbooks/gitlab/recipes/gitlab-shell.rb

@@ -72,6 +72,7 @@ templatesymlink "Create a config.yml and create a symlink to Rails root" do
               custom_hooks_dir: node['gitlab']['gitlab-shell']['custom_hooks_dir'],
               migration: node['gitlab']['gitlab-shell']['migration'],
             })
+  notifies :run, 'bash[Set proper security context on ssh files for selinux]', :delayed if SELinuxHelper.enabled?
 end
 link File.join(gitlab_shell_dir, ".gitlab_shell_secret") do
@@ -83,4 +84,5 @@ file authorized_keys do
   group git_group
   mode '600'
   action :create_if_missing
+  notifies :run, 'bash[Set proper security context on ssh files for selinux]', :delayed if SELinuxHelper.enabled?
 end

files/gitlab-cookbooks/gitlab/recipes/gitlab-workhorse.rb

@@ -16,11 +16,13 @@
 #
 account_helper = AccountHelper.new(node)
 redis_helper = RedisHelper.new(node)
+workhorse_helper = GitlabWorkhorseHelper.new(node)
 working_dir = node['gitlab']['gitlab-workhorse']['dir']
 log_directory = node['gitlab']['gitlab-workhorse']['log_directory']
 gitlab_workhorse_static_etc_dir = "/opt/gitlab/etc/gitlab-workhorse"
 workhorse_env_dir = node['gitlab']['gitlab-workhorse']['env_directory']
+gitlab_workhorse_socket_dir = node['gitlab']['gitlab-workhorse']['sockets_directory']
 directory working_dir do
   owner account_helper.gitlab_user
@@ -29,6 +31,16 @@ directory working_dir do
   recursive true
 end
+if workhorse_helper.unix_socket? && !gitlab_workhorse_socket_dir.nil?
+  directory gitlab_workhorse_socket_dir do
+    owner account_helper.gitlab_user
+    group account_helper.web_server_group
+    mode '0750'
+    notifies :restart, "runit_service[gitlab-workhorse]"
+    recursive true
+  end
+end
+
 directory log_directory do
   owner account_helper.gitlab_user
   mode '0700'
@@ -83,4 +95,5 @@ template config_file_path do
   mode "0640"
   variables(object_store: object_store, object_store_provider: object_store_provider, redis_url: redis_url, password: redis_password, sentinels: redis_sentinels, sentinel_master: redis_sentinel_master, master_password: redis_sentinel_master_password)
   notifies :restart, "runit_service[gitlab-workhorse]"
+  notifies :run, 'bash[Set proper security context on ssh files for selinux]', :delayed if SELinuxHelper.enabled?
 end

files/gitlab-cookbooks/gitlab/recipes/selinux.rb

@@ -27,12 +27,19 @@ if RedhatHelper.system_is_rhel7? || RedhatHelper.system_is_rhel8?
     not_if "getenforce | grep Disabled"
     not_if "semodule -l | grep '^#{authorized_keys_module}\\s'"
   end
+
+  gitlab_shell_module = 'gitlab-13.5.0-gitlab-shell'
+  execute "semodule -i /opt/gitlab/embedded/selinux/rhel/7/#{gitlab_shell_module}.pp" do
+    not_if "getenforce | grep Disabled"
+    not_if "semodule -l | grep '^#{gitlab_shell_module}\\s'"
+  end
 end
 # If SELinux is enabled, make sure that OpenSSH thinks the .ssh directory and authorized_keys file of the
 # git_user is valid.
 bash "Set proper security context on ssh files for selinux" do
-  code SELinuxHelper.commands(node)
+  code lazy { SELinuxHelper.commands(node) }
   only_if "id -Z"
   not_if { !node['gitlab']['gitlab-rails']['enable'] }
+  action :nothing
 end

files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb

@@ -267,7 +267,7 @@ production: &base
       remote_directory: <%= quote(@uploads_object_store_remote_directory) %>
       connection: <%= @uploads_object_store_connection.to_json %>
-  ## Packages (EE only)
+  ## Packages
   packages:
     enabled: <%= @packages_enabled %>
     # The location where build packages are stored (default: shared/packages).

files/gitlab-cookbooks/gitlab/templates/default/sv-gitlab-workhorse-run.erb

@@ -8,6 +8,13 @@ exec 2>&1
 cd <%= node['gitlab']['gitlab-workhorse']['dir'] %>
+<% if File.exist?('/var/opt/gitlab/gitlab-workhorse/socket') %>
+if [ -e "/var/opt/gitlab/gitlab-workhorse/socket" ]; then
+  echo "Removing orphaned workhorse socket at '/var/opt/gitlab/gitlab-workhorse/socket'"
+  rm /var/opt/gitlab/gitlab-workhorse/socket
+fi
+<% end %>
+
 exec chpst -e /opt/gitlab/etc/gitlab-workhorse/env -P \
   -U <%= node['gitlab']['user']['username'] %>:<%= node['gitlab']['user']['group'] %> \
   -u <%= node['gitlab']['user']['username'] %>:<%= node['gitlab']['user']['group'] %> \