Merge branch 'brodock/patroni-slots' into 'master'

Allow configuring permanent replication slots in patroni

Closes #5628

See merge request gitlab-org/omnibus-gitlab!4534

changelogs/unreleased/brodock-patroni-slots.yml

+---
+title: Allow configuring permanent replication slots in patroni
+merge_request: 4534
+author:
+type: added

files/gitlab-config-template/gitlab.rb.template

@@ -2451,6 +2451,11 @@ gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = nil
 # patroni['use_pg_rewind'] = false
 # patroni['use_slots'] = true
+## Permanent replication slots for Streaming Replication
+# patroni['replication_slots'] = {
+#   'geo_secondary' => { 'type' => 'physical' }
+# }
+
 ## The address and port that Patroni API binds to and listens on.
 # patroni['listen_address'] = nil
 # patroni['port'] = '8008'

files/gitlab-cookbooks/patroni/attributes/default.rb

@@ -22,6 +22,7 @@ default['patroni']['max_timelines_history'] = 0
 default['patroni']['master_start_timeout'] = 300
 default['patroni']['use_pg_rewind'] = false
 default['patroni']['use_slots'] = true
+default['patroni']['replication_slots'] = {}
 default['patroni']['postgresql']['wal_level'] = 'replica'
 default['patroni']['postgresql']['hot_standby'] = 'on'

files/gitlab-cookbooks/patroni/libraries/patroni_helper.rb

@@ -43,17 +43,26 @@ class PatroniHelper < BaseHelper
     dcs = {
       'postgresql' => {
         'parameters' => {}
-      }
+      },
+      'slots' => {}
     }
+
     DCS_ATTRIBUTES.each do |key|
       dcs[key] = node['patroni'][key]
     end
+
     DCS_POSTGRESQL_ATTRIBUTES.each do |key|
       dcs['postgresql'][key] = node['patroni'][key]
     end
+
     node['patroni']['postgresql'].each do |key, value|
       dcs['postgresql']['parameters'][key] = value
     end
+
+    node['patroni']['replication_slots'].each do |slot_name, options|
+      dcs['slots'][slot_name] = parse_replication_slots_options(options)
+    end
+
     dcs
   end
@@ -69,4 +78,17 @@ class PatroniHelper < BaseHelper
       }
     }
   end
+
+  private
+
+  # Parse replication slots attributes
+  #
+  # We currently support only physical replication
+  def parse_replication_slots_options(options)
+    return unless options['type'] == 'physical'
+
+    {
+      'type' => 'physical'
+    }
+  end
 end

spec/chef/gitlab-ee/recipes/patroni_spec.rb

@@ -96,6 +96,7 @@ RSpec.describe 'patroni cookbook' do
                 checkpoint_timeout: 30,
               },
             },
+            slots: {},
           },
           method: 'gitlab_ctl',
           gitlab_ctl: {
@@ -170,6 +171,9 @@ RSpec.describe 'patroni cookbook' do
           use_pg_rewind: true,
           connect_address: '1.2.3.4',
           connect_port: 18008,
+          replication_slots: {
+            'geo_secondary' => { 'type' => 'physical' }
+          },
           consul: {
             service_check_interval: '20s'
           },
@@ -185,6 +189,7 @@ RSpec.describe 'patroni cookbook' do
     it 'should be reflected in patroni configuration file' do
       expect(chef_run).to render_file('/var/opt/gitlab/patroni/patroni.yaml').with_content { |content|
         cfg = YAML.safe_load(content, permitted_classes: [Symbol], symbolize_names: true)
+
         expect(cfg).to include(
           name: 'test-node-name',
           scope: 'test-scope',
@@ -207,7 +212,10 @@ RSpec.describe 'patroni cookbook' do
         expect(cfg[:bootstrap][:dcs]).to include(
           loop_wait: 20,
           ttl: 60,
-          master_start_timeout: 600
+          master_start_timeout: 600,
+          slots: {
+            geo_secondary: { type: 'physical' }
+          }
         )
         expect(cfg[:bootstrap][:dcs][:postgresql]).to include(
           use_slots: false,
@@ -220,6 +228,30 @@ RSpec.describe 'patroni cookbook' do
         )
       }
     end
+
+    it 'should reflect into dcs config file' do
+      expect(chef_run).to render_file('/var/opt/gitlab/patroni/dcs.yaml').with_content { |content|
+        cfg = YAML.safe_load(content, permitted_classes: [Symbol], symbolize_names: true)
+
+        expect(cfg).to include(
+          loop_wait: 20,
+          ttl: 60,
+          master_start_timeout: 600,
+          slots: {
+            geo_secondary: { type: 'physical' }
+          }
+        )
+        expect(cfg[:postgresql]).to include(
+          use_slots: false,
+          use_pg_rewind: true
+        )
+        expect(cfg[:postgresql][:parameters]).to include(
+          wal_keep_segments: 16,
+          max_wal_senders: 4,
+          max_replication_slots: 4
+        )
+      }
+    end
   end
   context 'when building a cluster' do