Add Patroni runtime

This also includes Psycopg2 which is required by both patroni service
and patronictl commands.

.custom_sources.yml

@@ -124,3 +124,6 @@ ohai:
 redis:
   remote: "git@dev.gitlab.org:omnibus-mirror/redis.git"
   alternative: "https://gitlab.com/gitlab-org/build/omnibus-mirror/redis.git"
+psycopg2:
+  remote: "git@dev.gitlab.org:omnibus-mirror/psycopg2.git"
+  alternative: "https://gitlab.com/gitlab-org/build/omnibus-mirror/psycopg2.git"

changelogs/unreleased/5112-patroni-omnibus.yml

+---
+title: Add Patroni to Omnibus
+merge_request: 3984
+author:
+type: added

config/patches/patroni/add-license-file.patch

+diff --git a/LICENSE b/LICENSE
+new file mode 100644
+index 0000000..e06d208
+--- /dev/null
++++ b/LICENSE
+@@ -0,0 +1,21 @@
++The MIT License (MIT)
++
++Copyright (c) 2015 Compose, Zalando SE
++
++Permission is hereby granted, free of charge, to any person obtaining a copy
++of this software and associated documentation files (the "Software"), to deal
++in the Software without restriction, including without limitation the rights
++to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
++copies of the Software, and to permit persons to whom the Software is
++furnished to do so, subject to the following conditions:
++
++The above copyright notice and this permission notice shall be included in all
++copies or substantial portions of the Software.
++
++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
++AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++SOFTWARE.

config/software/gitlab-rails.rb

@@ -56,6 +56,7 @@ dependency 'exiftool'
 if EE
   dependency 'pgbouncer'
   dependency 'repmgr'
+  dependency 'patroni'
   dependency 'gitlab-elasticsearch-indexer'
 end

config/software/patroni.rb

+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name 'patroni'
+default_version '1.6.4'
+
+license 'MIT'
+license_file 'LICENSE'
+
+skip_transitive_dependency_licensing true
+
+dependency 'python3'
+dependency 'postgresql'
+dependency 'psycopg2'
+
+build do
+  env = with_standard_compiler_flags(with_embedded_path)
+
+  patch source: "add-license-file.patch"
+
+  command "#{install_dir}/embedded/bin/pip3 install patroni[consul]==#{version}", env: env
+end

config/software/psycopg2.rb

+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name 'psycopg2'
+version = Gitlab::Version.new('psycopg2', '2_8_4')
+default_version version.print(false)
+
+license 'LGPL'
+license_file 'LICENSE'
+
+skip_transitive_dependency_licensing true
+
+source git: version.remote
+
+dependency 'python3'
+dependency 'postgresql'
+
+pg_major_version = '11'
+
+build do
+  env = with_standard_compiler_flags(with_embedded_path)
+
+  # Prepend PostgreSQL's bin directory to the PATH, so that setup.py can find pg_config and build against it
+  env['PATH'] = "#{install_dir}/embedded/postgresql/#{pg_major_version}/bin:#{env['PATH']}"
+
+  command "#{install_dir}/embedded/bin/python3 setup.py build_ext", env: env
+  command "#{install_dir}/embedded/bin/python3 setup.py install", env: env
+end

config/software/python3.rb

@@ -32,6 +32,7 @@ dependency 'openssl'
 dependency 'bzip2'
 dependency 'libffi'
 dependency 'liblzma'
+dependency 'libyaml'
 license 'Python-2.0'
 license_file 'LICENSE'

doc/package-information/defaults.md

@@ -50,6 +50,7 @@ by default:
 | <a name="mattermost-web"></a>      Mattermost          | No            | Port             | X           | 80 or 443                              |
 | <a name="pgbouncer"></a>           PgBouncer           | No            | Port             | X           | 6432                                   |
 | <a name="consul"></a>              Consul              | No            | Port             | X           | 8300, 8500                             |
+| <a name="patroni"></a>             Patroni             | No            | Port             | X           | 8008                                   |
 Legend:

files/gitlab-config-template/gitlab.rb.template

@@ -938,6 +938,12 @@ external_url 'GENERATED_EXTERNAL_URL'
 # postgresql['enable'] = true
 # postgresql['listen_address'] = nil
 # postgresql['port'] = 5432
+
+## Only used when Patroni is enabled. This is the port that PostgreSQL responds to other
+## cluster members. This port is used by Patroni to advertize the PostgreSQL connection
+## endpoint to the cluster. By default it is the same as postgresql['port'].
+# postgresql['connect_port'] = 5432
+
 # postgresql['data_dir'] = "/var/opt/gitlab/postgresql/data"
 ##! **recommend value is 1/4 of total RAM, up to 14GB.**
@@ -2350,6 +2356,63 @@ external_url 'GENERATED_EXTERNAL_URL'
 # repmgr['daemon'] = true
 # repmgrd['enable'] = true
+################################################################################
+# Patroni (EE only)
+#
+# NOTICE: Patroni is an experimental feature and subject to change.
+#
+################################################################################
+# patroni['enable'] = false
+
+# patroni['dir'] = '/var/opt/gitlab/patroni'
+# patroni['data_dir'] = '/var/opt/gitlab/patroni/data'
+# patroni['ctl_command'] = '/opt/gitlab/embedded/bin/patronictl'
+
+# patroni['scope'] = 'gitlab-postgresql-ha'
+# patroni['name'] = nil
+
+# patroni['log_directory'] = '/var/log/gitlab/patroni'
+# patroni['log_level'] = 'INFO'
+
+# patroni['consul']['url'] = 'http://127.0.0.1:8500'
+# patroni['consul']['service_check_interval'] = '10s'
+# patroni['consul']['register_service'] = false
+# patroni['consul']['checks'] = []
+
+## Bootstrap settings
+# patroni['loop_wait'] = 10
+# patroni['ttl'] = 30
+# patroni['retry_timeout'] = 10
+# patroni['maximum_lag_on_failover'] = 1_048_576
+# patroni['max_timelines_history'] = 0
+# patroni['master_start_timeout'] = 300
+
+## PostgreSQL configuration override
+# patroni['postgresql']['wal_level'] = 'replica'
+# patroni['postgresql']['hot_standby'] = 'on'
+# patroni['postgresql']['wal_keep_segments'] = 8
+# patroni['postgresql']['max_wal_senders'] = 5
+# patroni['postgresql']['max_replication_slots'] = 5
+# patroni['postgresql']['checkpoint_timeout'] = 30
+
+# patroni['use_pg_rewind'] = false
+# patroni['use_slots'] = true
+
+## The address and port that Patroni API binds to and listens on.
+# patroni['listen_address'] = nil
+# patroni['port'] = '8008'
+
+## The address of the Patroni node that is advertized to other cluster
+## members to communicate with its API and PostgreSQL. If it is not specified,
+## it tries to use the first available private IP and falls back to the default
+## network interface.
+# patroni['connect_address'] = nil
+
+## The port that Patroni API responds to other cluster members. This port is
+## advertized and by default is the same as patroni['port'].
+# patroni['connect_port'] = '8008'
+
+
 ################################################################################
 # Consul (EEP only)
 ################################################################################

files/gitlab-cookbooks/consul/attributes/services.rb

 default['consul']['services'] = []
-default['consul']['service_config'] = {
-  'postgresql' => {
-    'service' => {
-      'name' => "postgresql",
-      'address' => '',
-      'port' => 5432,
-      'check' => {
-        'id': 'service:postgresql',
-        'args' => ['/opt/gitlab/bin/gitlab-ctl', 'repmgr-check-master'],
-        'interval' => "10s",
-        'status': 'failing'
-      }
-    },
-    'watches': [
-      {
-        'type': 'keyprefix',
-        'prefix': 'gitlab/ha/postgresql/failed_masters/',
-        'args': ['/opt/gitlab/bin/gitlab-ctl', 'consul', 'watchers', 'handle-failed-master']
-      }
-    ]
+default['consul']['service_config'] = nil
+
+default['consul']['internal']['postgresql_service_name'] = 'postgresql'
+default['consul']['internal']['postgresql_service_check_interval'] = '10s'
+default['consul']['internal']['postgresql_service_check_status'] = 'failing'
+default['consul']['internal']['postgresql_service_check_args_repmgr'] = ['/opt/gitlab/bin/gitlab-ctl', 'repmgr-check-master']
+default['consul']['internal']['postgresql_service_check_args_patroni'] = ['/opt/gitlab/bin/gitlab-ctl', 'patroni', 'check-leader']
+default['consul']['internal']['postgresql_watches_repmgr'] = [
+  {
+    'type': 'keyprefix',
+    'prefix': 'gitlab/ha/postgresql/failed_masters/',
+    'args': ['/opt/gitlab/bin/gitlab-ctl', 'consul', 'watchers', 'handle-failed-master']
   }
-}
+]

files/gitlab-cookbooks/consul/libraries/consul_helper.rb

@@ -45,4 +45,27 @@ class ConsulHelper
     end
     config.to_json
   end
+
+  def postgresql_service_config
+    return node['consul']['service_config']['postgresql'] || {} unless node['consul']['service_config'].nil?
+
+    ha_solution = Gitlab['patroni']['enable'] ? 'patroni' : 'repmgr'
+    cfg = {
+      'service' => {
+        'name' => node['consul']['internal']['postgresql_service_name'],
+        'address' => '',
+        'port' => node['postgresql']['port'],
+        'check' => {
+          'id': "service:#{node['consul']['internal']['postgresql_service_name']}",
+          'interval' => node['consul']['internal']['postgresql_service_check_interval'],
+          'status': node['consul']['internal']['postgresql_service_check_status'],
+          'args': node['consul']['internal']["postgresql_service_check_args_#{ha_solution}"]
+        }
+      }
+    }
+
+    cfg['watches'] = node['consul']['internal']['postgresql_watches_repmgr'] if ha_solution == 'repmgr'
+
+    cfg
+  end
 end

files/gitlab-cookbooks/consul/recipes/service_postgresql.rb

@@ -14,10 +14,13 @@
 # limitations under the License.
 #
 account_helper = AccountHelper.new(node)
+pg_helper = PgHelper.new(node)
+consul_helper = ConsulHelper.new(node)
 file "#{node['consul']['config_dir']}/postgresql_service.json" do
-  content node['consul']['service_config']['postgresql'].to_json
+  content consul_helper.postgresql_service_config.to_json
   owner account_helper.consul_user
+  notifies :run, 'execute[reload consul]', :delayed
 end
-include_recipe 'repmgr::consul_user_permissions' if node['repmgr']['master_on_initialization']
+include_recipe 'repmgr::consul_user_permissions' if !pg_helper.delegated? && node['repmgr']['master_on_initialization']

files/gitlab-cookbooks/gitlab-ee/metadata.rb

@@ -14,3 +14,4 @@ depends "package"
 depends "gitlab"
 depends 'consul'
 depends 'repmgr'
+depends 'patroni'

files/gitlab-cookbooks/gitlab-ee/recipes/default.rb

@@ -36,6 +36,7 @@ end
 %w(
   consul
   repmgr
+  patroni
 ).each do |service|
   if node[service]['enable']
     include_recipe "#{service}::enable"

files/gitlab-cookbooks/gitlab/libraries/helpers/base_pg_helper.rb

@@ -214,6 +214,8 @@ class BasePgHelper < BaseHelper
               "|grep -x t"])
   end
+  alias_method :is_replica?, :is_slave?
+
   def is_offline_or_readonly?
     !is_running? || is_slave?
   end
@@ -298,6 +300,41 @@ class BasePgHelper < BaseHelper
     raise NotImplementedError
   end
+  def delegated?
+    # When Patroni is enabled, the configuration of PostgreSQL instance must be delegated to it.
+    # PostgreSQL cookbook skips some of the steps that are must be done either during or after
+    # Patroni bootstraping.
+    Gitlab['patroni']['enable'] && !Gitlab['repmgr']['enable']
+  end
+
+  def config_dir
+    Gitlab['patroni']['enable'] ? node['patroni']['data_dir'] : node['postgresql']['data_dir']
+  end
+
+  def postgresql_config
+    ::File.join(config_dir, "postgresql#{Gitlab['patroni']['enable'] ? '.base' : ''}.conf")
+  end
+
+  def postgresql_runtime_config
+    ::File.join(config_dir, 'runtime.conf')
+  end
+
+  def pg_hba_config
+    ::File.join(config_dir, 'pg_hba.conf')
+  end
+
+  def pg_ident_config
+    ::File.join(config_dir, 'pg_ident.conf')
+  end
+
+  def ssl_cert_file
+    ::File.join(config_dir, node['postgresql']['ssl_cert_file'])
+  end
+
+  def ssl_key_file
+    ::File.join(config_dir, node['postgresql']['ssl_key_file'])
+  end
+
   private
   def stringify_hash_values(options)

files/gitlab-cookbooks/gitlab/libraries/patroni.rb

+require 'socket'
+
+module Patroni
+  class << self
+    def parse_variables
+      return unless Services.enabled?('patroni')
+
+      Gitlab['patroni']['connect_address'] ||= private_ipv4 || Gitlab['node']['ipaddress']
+      Gitlab['patroni']['connect_port'] ||= Gitlab['patroni']['port'] || Gitlab['node']['patroni']['port']
+    end
+
+    def private_ipv4
+      Socket.getifaddrs.select { |ifaddr| ifaddr.addr.ipv4_private? }.first&.addr&.ip_address
+    end
+  end
+end

files/gitlab-cookbooks/gitlab/libraries/postgresql.rb

@@ -19,6 +19,7 @@ module Postgresql
   class << self
     def parse_variables
       parse_postgresql_settings
+      parse_connect_port
       parse_multi_db_host_addresses
       parse_mattermost_postgresql_settings
     end
@@ -81,6 +82,10 @@ module Postgresql
       Gitlab['mattermost']['sql_data_source'] = value_from_gitlab_rb || value_from_attributes
     end
+    def parse_connect_port
+      Gitlab['postgresql']['connect_port'] ||= Gitlab['postgresql']['port'] || Gitlab['node']['postgresql']['port']
+    end
+
     def postgresql_managed?
       Services.enabled?('postgresql')
     end

files/gitlab-cookbooks/package/libraries/config/gitlab.rb

@@ -48,6 +48,7 @@ module Gitlab
   attribute('repmgr')
   attribute('repmgrd')
   attribute('consul')
+  attribute('patroni').use { Patroni }
   attribute('gitaly').use { Gitaly }
   attribute('praefect').use { Praefect }
   attribute('mattermost',  priority: 30).use { GitlabMattermost } # Mattermost checks if GitLab is enabled on the same box

files/gitlab-cookbooks/package/libraries/config/roles/postgres.rb

@@ -18,7 +18,7 @@ module PostgresRole
   def self.load_role
     return unless Gitlab['postgres_role']['enable']
-    Gitlab['repmgr']['enable'] = true if Gitlab['repmgr']['enable'].nil?
+    Gitlab['repmgr']['enable'] = true if !Gitlab['patroni']['enable'] && Gitlab['repmgr']['enable'].nil?
     Services.enable_group('postgres_role')
   end
 end

files/gitlab-cookbooks/package/libraries/config/services.rb

@@ -54,6 +54,7 @@ module Services
     service 'pgbouncer',          groups: %w(postgres pgbouncer_role)
     service 'pgbouncer_exporter', groups: %w(pgbouncer_role)
     service 'repmgrd',            groups: %w(postgres postgres_role)
+    service 'patroni',            groups: %w(postgres)
     service 'consul',             groups: %w(consul_role ha pgbouncer_role postgres_role)
     service 'sidekiq_cluster',    groups: ['sidekiq']
   end