Merge branch 'pitr' into 'master'

Implementation of PITR recovery before promotion

Closes gitlab#225173

See merge request gitlab-org/omnibus-gitlab!4636

(cherry picked from commit 9ae9fcf7)

691a1fa5 Geo: Implement PITR recovery before promotion of secondary node
1171279d Apply 1 suggestion(s) to 1 file(s)
402e18c6 Apply suggestion. Better lsn file check
fe0c04a3 Merge branch 'master' of gitlab.com:gitlab-org/omnibus-gitlab into pitr
e3f99908 Apply 1 suggestion(s) to 1 file(s)
1d5781dc Merge branch 'pitr' of gitlab.com:gitlab-org/omnibus-gitlab into pitr
7a34d0da Apply 1 suggestion(s) to 1 file(s)
3e9c8592 Apply suggestions
dbbccdee Merge branch 'pitr' of gitlab.com:gitlab-org/omnibus-gitlab into pitr
2ff5b783 Apply suggestions
8b9163ac Add suggestions
565f96c6 Apply suggestions
39b2465d Apply suggestions
c8f5d0ab Merge remote-tracking branch 'origin/master' into pitr
0ba306f4 Fix part of Geo::PromoteToPrimaryNode specs
90c50385 Make recovery_to_point_in_time tio return true
8dd17632 Merge remote-tracking branch 'origin/master' into pitr
1af3bf5a Separate Promote DB task
8a521139 Update specs for the command split
f068dfc3 Merge remote-tracking branch 'origin/master' into pitr
175a4efd Apply 1 suggestion(s) to 1 file(s)
b1af4039 Remove io/console

changelogs/unreleased/pitr.yml

+---
+title: 'Geo: Perform point-in-time recovery before promotion of secondary node'
+merge_request: 4636
+author:
+type: fixed

files/gitlab-ctl-commands-ee/lib/geo/promote_db.rb

+require 'rainbow/ext/string'
+
+module Geo
+  # PromoteDb promotes standby database as usual "pg-ctl promote" but
+  # if point-in-time LSN file is found, the database will be recovered to that state first
+  class PromoteDb
+    PITR_FILE_NAME = 'geo-pitr-file'.freeze
+
+    attr_accessor :base_path, :data_path
+
+    def initialize(ctl)
+      @base_path = ctl.base_path
+      @data_path = ctl.data_path
+    end
+
+    def execute
+      return true if recovery_to_point_in_time
+
+      puts
+      puts 'Promoting the PostgreSQL read-only replica to primary...'.color(:yellow)
+      puts
+
+      run_command('/opt/gitlab/embedded/bin/gitlab-pg-ctl promote', live: true).error!
+
+      success_message
+    end
+
+    private
+
+    def postgresql_version
+      @postgresql_version ||= GitlabCtl::PostgreSQL.postgresql_version(data_path)
+    end
+
+    def recovery_to_point_in_time
+      lsn = lsn_from_pitr_file
+
+      return if lsn.nil?
+
+      puts
+      puts "Recovery to point #{lsn} and promoting...".color(:yellow)
+      puts
+
+      write_recovery_settings(lsn)
+
+      run_command('gitlab-ctl restart postgresql', live: true).error!
+
+      success_message
+
+      true
+    end
+
+    def lsn_from_pitr_file
+      geo_pitr_file = "#{data_path}/postgresql/data/#{PITR_FILE_NAME}"
+
+      return nil unless File.exist?(geo_pitr_file)
+
+      lsn = File.read(geo_pitr_file)
+
+      lsn.empty? ? nil : lsn
+    end
+
+    def built_recovery_setting_for_pitr(lsn)
+      <<-EOF
+        recovery_target_lsn = '#{lsn}'
+        recovery_target_action = 'promote'
+      EOF
+    end
+
+    def write_recovery_settings(lsn)
+      settings = built_recovery_setting_for_pitr(lsn)
+
+      if postgresql_version >= 12
+        puts "PostgreSQL 12 or newer. Writing settings to postgresql.conf...".color(:green)
+
+        write_geo_config_file(settings)
+      else
+        puts "Writing recovery.conf...".color(:green)
+
+        write_recovery_conf(settings)
+      end
+    end
+
+    def write_geo_config_file(settings)
+      geo_conf_file = "#{data_path}/postgresql/data/gitlab-geo.conf"
+
+      File.open(geo_conf_file, "w", 0640) do |file|
+        file.write(settings)
+      end
+    end
+
+    def write_recovery_conf(settings)
+      recovery_conf = "#{data_path}/postgresql/data/recovery.conf"
+
+      File.open(recovery_conf, 'a', 0640) do |file|
+        file.write(settings)
+      end
+    end
+
+    def run_command(cmd, live: false)
+      GitlabCtl::Util.run_command(cmd, live: live)
+    end
+
+    def success_message
+      puts
+      puts 'The database is successfully promoted!'.color(:green)
+    end
+  end
+end

files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node.rb

@@ -3,8 +3,11 @@ require 'rainbow/ext/string'
 module Geo
   class PromoteToPrimaryNode
-    def initialize(base_path, options)
-      @base_path = base_path
+    attr_accessor :ctl, :base_path
+
+    def initialize(ctl, options)
+      @ctl = ctl
+      @base_path = @ctl.base_path
       @options = options
     end
@@ -61,11 +64,7 @@ module Geo
     end
     def promote_postgresql_to_primary
-      puts
-      puts 'Promoting the PostgreSQL to primary...'.color(:yellow)
-      puts
-
-      run_command('/opt/gitlab/embedded/bin/gitlab-pg-ctl promote', live: true).error!
+      Geo::PromoteDb.new(ctl).execute
     end
     def reconfigure

files/gitlab-ctl-commands-ee/lib/geo/replication_toggle_command.rb

 require_relative "./replication_process"
+require_relative "./promote_db"
 require 'optparse'
 require 'English'
@@ -19,6 +20,8 @@ module Geo
     def execute!
       @replication_process.send(@action.to_sym)
+
+      process_pitr_file
     rescue Geo::PsqlError => e
       puts "Postgres encountered an error: #{e.message}"
       exit 1
@@ -33,6 +36,30 @@ module Geo
     private
+    attr_reader :action, :ctl
+
+    def process_pitr_file
+      geo_pitr_file_path = "#{ctl.data_path}/postgresql/data/#{Geo::PromoteDb::PITR_FILE_NAME}"
+
+      if action == 'pause'
+        puts "* Create Geo point-in-time recovery file".color(:green)
+
+        File.write(geo_pitr_file_path, current_lsn)
+      elsif action == 'resume'
+        puts "* Remove Geo point-in-time recovery file".color(:green)
+
+        File.delete(geo_pitr_file_path) if File.exist?(geo_pitr_file_path)
+      end
+    end
+
+    def current_lsn
+      run_query('SELECT pg_last_wal_replay_lsn()')
+    end
+
+    def run_query(query)
+      GitlabCtl::Util.get_command_output("gitlab-psql -d postgres -c '#{query}' -q -t").strip
+    end
+
     def parse_options!
       opts_parser = OptionParser.new do |opts|
         opts.banner = "Usage: gitlab-ctl replication-process-#{@action} [options]"

files/gitlab-ctl-commands-ee/promote_db.rb

+require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/geo/promote_db"
+
+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_command_under_category('promote-db', 'gitlab-geo', 'Promote secondary PostgreSQL database', 2) do |cmd_name, *args|
+  Geo::PromoteDb.new(self).execute
+end

files/gitlab-ctl-commands-ee/promote_to_primary_node.rb

@@ -18,26 +18,26 @@ require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/geo/promote_to_primary
 #
 add_command_under_category('promote-to-primary-node', 'gitlab-geo', 'Promote to primary node', 2) do |cmd_name, *args|
-  Geo::PromoteToPrimaryNode.new(base_path, get_ctl_options).execute
-end
-def get_ctl_options
-  options = {}
-  OptionParser.new do |opts|
-    opts.banner = "Usage: gitlab-ctl promote-to-primary-node [options]"
-    opts.on('-p', '--[no-]confirm-primary-is-down', 'Do not ask for confirmation that primary is down') do |p|
-      options[:confirm_primary_is_down] = p
-    end
-    opts.on('-m', '--skip-preflight-checks', 'Perform promotion without running any preflight checks') do |m|
-      options[:skip_preflight_checks] = m
-    end
-    opts.on('-f', '--force', 'Proceed even if preflight checks fail') do |f|
-      options[:force] = f
-    end
-  end.parse!(ARGV.dup)
-  options
+  def get_ctl_options
+    options = {}
+    OptionParser.new do |opts|
+      opts.banner = "Usage: gitlab-ctl promote-to-primary-node [options]"
+      opts.on('-p', '--[no-]confirm-primary-is-down', 'Do not ask for confirmation that primary is down') do |p|
+        options[:confirm_primary_is_down] = p
+      end
+      opts.on('-m', '--skip-preflight-checks', 'Perform promotion without running any preflight checks') do |m|
+        options[:skip_preflight_checks] = m
+      end
+      opts.on('-f', '--force', 'Proceed even if preflight checks fail') do |f|
+        options[:force] = f
+      end
+    end.parse!(ARGV.dup)
+    options
+  end
+  Geo::PromoteToPrimaryNode.new(self, get_ctl_options).execute
 end

spec/gitlab-ctl-commands-ee/lib/geo/promote_db_spec.rb

+require 'spec_helper'
+require 'geo/promote_db'
+require 'gitlab_ctl/util'
+
+RSpec.describe Geo::PromoteDb, '#execute' do
+  let(:instance) { double(base_path: '/opt/gitlab/embedded', data_path: '/var/opt/gitlab/postgresql/data') }
+
+  subject(:command) { described_class.new(instance) }
+
+  before do
+    allow($stdout).to receive(:puts)
+    allow($stdout).to receive(:print)
+
+    allow(command).to receive(:run_command).with(any_args)
+
+    allow(command).to receive(:run_command).and_return(double('error!' => nil))
+  end
+
+  context 'when PITR file does not exist' do
+    it 'does not run PITR recovery' do
+      expect(command).not_to receive(:write_recovery_settings)
+
+      command.execute
+    end
+  end
+
+  context 'when PITR file exists' do
+    let(:lsn) { '16/B374D848' }
+
+    before do
+      allow(command).to receive(:lsn_from_pitr_file).and_return(lsn)
+    end
+
+    it 'runs PITR recovery' do
+      expect(command).to receive(:write_recovery_settings).with(lsn)
+
+      expect { command.execute }.to output(
+        /Recovery to point #{lsn} and promoting.../).to_stdout
+    end
+
+    context 'PG version 11' do
+      it 'runs PITR recovery' do
+        allow(command).to receive(:postgresql_version).and_return(11)
+        expect(command).to receive(:write_recovery_conf)
+
+        expect { command.execute }.to output(
+          /Writing recovery.conf/).to_stdout
+      end
+    end
+
+    context 'PG version 12' do
+      it 'runs PITR recovery' do
+        allow(command).to receive(:postgresql_version).and_return(12)
+        expect(command).to receive(:write_geo_config_file)
+
+        expect { command.execute }.to output(
+          /PostgreSQL 12 or newer. Writing settings to postgresql.conf/).to_stdout
+      end
+    end
+  end
+end

spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node_spec.rb

 require 'spec_helper'
 require 'fileutils'
 require 'geo/promote_to_primary_node'
+require 'geo/promote_db'
 require 'geo/promotion_preflight_checks'
 require 'gitlab_ctl/util'
 RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
   let(:options) { { skip_preflight_checks: true } }
-  subject(:command) { described_class.new(nil, options) }
+  let(:instance) { double(base_path: '/opt/gitlab/embedded', data_path: '/var/opt/gitlab/postgresql/data') }
+
+  subject(:command) { described_class.new(instance, options) }
   let(:temp_directory) { Dir.mktmpdir }
   let(:gitlab_config_path) { File.join(temp_directory, 'gitlab.rb') }
@@ -23,6 +26,23 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
     FileUtils.rm_rf(temp_directory)
   end
+  describe '#promote_postgresql_to_primary' do
+    before do
+      allow(STDIN).to receive(:gets).and_return('y')
+
+      allow(command).to receive(:toggle_geo_roles).and_return(true)
+      allow(command).to receive(:reconfigure).and_return(true)
+      allow(command).to receive(:promote_to_primary).and_return(true)
+      allow(command).to receive(:success_message).and_return(true)
+    end
+
+    it 'promotes the database' do
+      expect_any_instance_of(Geo::PromoteDb).to receive(:execute)
+
+      command.execute
+    end
+  end
+
   describe '#run_preflight_checks' do
     before do
       allow(STDIN).to receive(:gets).and_return('y')
@@ -57,7 +77,7 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
       it 'passes given options to preflight checks command' do
         expect(Geo::PromotionPreflightChecks).to receive(:new).with(
-          nil, options).and_call_original
+          '/opt/gitlab/embedded', options).and_call_original
         command.execute
       end

spec/gitlab-ctl-commands-ee/lib/geo/replication_toggle_command_spec.rb

@@ -4,19 +4,26 @@ $LOAD_PATH << './files/gitlab-ctl-commands-ee/lib'
 $LOAD_PATH << './files/gitlab-ctl-commands/lib'
 require 'geo/replication_toggle_command'
+require 'geo/promote_to_primary_node'
+require 'gitlab_ctl/util'
 RSpec.describe Geo::ReplicationToggleCommand do
   let(:status) { double('Command status', error?: false) }
   let(:arguments) { [] }
-  let(:ctl_instance) { double('gitlab-ctl instance', base_path: '') }
+  let(:ctl_instance) { double('gitlab-ctl instance', base_path: '', data_path: 'data_path') }
+
+  before do
+    allow_any_instance_of(Geo::ReplicationToggleCommand).to receive(:current_lsn).and_return('16/B374D848')
+  end
   describe 'pause' do
     subject { described_class.new(ctl_instance, 'pause', arguments) }
     it 'calls pause' do
       expect_any_instance_of(Geo::ReplicationProcess).to receive(:pause)
-      subject.execute!
+      expect(File).to receive(:write).with('data_path/postgresql/data/geo-pitr-file', '16/B374D848')
+      expect { subject.execute! }.to output(/Create Geo point-in-time recovery file/).to_stdout
     end
     it 'rescues and exits if postgres has an error' do
@@ -33,8 +40,9 @@ RSpec.describe Geo::ReplicationToggleCommand do
       it 'uses the specified database' do
         expect(Geo::ReplicationProcess).to receive(:new).with(any_args, { db_name: 'database_i_want' }).and_call_original
         expect_any_instance_of(Geo::ReplicationProcess).to receive(:pause)
-        subject.execute!
+        expect(File).to receive(:write).with('data_path/postgresql/data/geo-pitr-file', '16/B374D848')
+        expect { subject.execute! }.to output(/Create Geo point-in-time recovery file/).to_stdout
       end
     end
   end
@@ -45,7 +53,7 @@ RSpec.describe Geo::ReplicationToggleCommand do
     it 'calls resume' do
       expect_any_instance_of(Geo::ReplicationProcess).to receive(:resume)
-      subject.execute!
+      expect { subject.execute! }.to output(/Remove Geo point-in-time recovery file/).to_stdout
     end
     it 'rescues and exits if postgres has an error' do
@@ -71,7 +79,7 @@ RSpec.describe Geo::ReplicationToggleCommand do
         expect(Geo::ReplicationProcess).to receive(:new).with(any_args, { db_name: 'database_i_want' }).and_call_original
         expect_any_instance_of(Geo::ReplicationProcess).to receive(:resume)
-        subject.execute!
+        expect { subject.execute! }.to output(/Remove Geo point-in-time recovery file/).to_stdout
       end
     end
   end

spec/gitlab-ctl-commands-ee/promote_db_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'geo/promote_db'
+
+RSpec.describe 'gitlab-ctl promote-db' do
+  let(:klass) { Geo::PromoteDb }
+  let(:command_name) { 'promote-db' }
+  let(:command_script) { 'promote_db' }
+
+  include_context 'ctl'
+
+  it_behaves_like 'gitlab geo promotion commands', 'promote-db'
+end