Geo: Implement PITR recovery before promotion of secondary node

If replication was paused on secondary, we need to use the last point
in time for the database when we promote the node.

changelogs/unreleased/pitr.yml

+---
+title: 'Geo: Perform point-in-time recovery before promotion of secondary node'
+merge_request: 4636
+author:
+type: fixed

files/gitlab-ctl-commands-ee/lib/geo/promote_db.rb

+require 'rainbow/ext/string'
+
+module Geo
+  # PromoteDb promotes standby database as usual "pg-ctl promote" but
+  # if point-in-time LSN file is found, the database will be recovered to that state first
+  class PromoteDb
+    PITR_FILE_NAME = 'geo-pitr-file'.freeze
+
+    attr_accessor :base_path, :data_path
+
+    def initialize(ctl)
+      @base_path = ctl.base_path
+      @data_path = ctl.data_path
+    end
+
+    def execute
+      return true if recovery_to_point_in_time
+
+      puts
+      puts 'Promoting the PostgreSQL read-only replica to primary...'.color(:yellow)
+      puts
+
+      run_command('/opt/gitlab/embedded/bin/gitlab-pg-ctl promote', live: true).error!
+
+      success_message
+    end
+
+    private
+
+    def postgresql_version
+      @postgresql_version ||= GitlabCtl::PostgreSQL.postgresql_version(data_path)
+    end
+
+    def recovery_to_point_in_time
+      lsn = lsn_from_pitr_file
+
+      return if lsn.nil?
+
+      puts
+      puts "Recovery to point #{lsn} and promoting...".color(:yellow)
+      puts
+
+      write_recovery_settings(lsn)
+
+      run_command('gitlab-ctl restart postgresql', live: true).error!
+
+      success_message
+
+      true
+    end
+
+    def lsn_from_pitr_file
+      geo_pitr_file = "#{data_path}/postgresql/data/#{PITR_FILE_NAME}"
+
+      return nil unless File.exist?(geo_pitr_file)
+
+      lsn = File.read(geo_pitr_file)
+
+      lsn.empty? ? nil : lsn
+    end
+
+    def built_recovery_setting_for_pitr(lsn)
+      <<-EOF
+        recovery_target_lsn = '#{lsn}'
+        recovery_target_action = 'promote'
+      EOF
+    end
+
+    def write_recovery_settings(lsn)
+      settings = built_recovery_setting_for_pitr(lsn)
+
+      if postgresql_version >= 12
+        puts "PostgreSQL 12 or newer. Writing settings to postgresql.conf...".color(:green)
+
+        write_geo_config_file(settings)
+      else
+        puts "Writing recovery.conf...".color(:green)
+
+        write_recovery_conf(settings)
+      end
+    end
+
+    def write_geo_config_file(settings)
+      geo_conf_file = "#{data_path}/postgresql/data/gitlab-geo.conf"
+
+      File.open(geo_conf_file, "w", 0640) do |file|
+        file.write(settings)
+      end
+    end
+
+    def write_recovery_conf(settings)
+      recovery_conf = "#{data_path}/postgresql/data/recovery.conf"
+
+      File.open(recovery_conf, 'a', 0640) do |file|
+        file.write(settings)
+      end
+    end
+
+    def run_command(cmd, live: false)
+      GitlabCtl::Util.run_command(cmd, live: live)
+    end
+
+    def success_message
+      puts
+      puts 'The database is successfully promoted!'.color(:green)
+    end
+  end
+end

files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node.rb

@@ -3,8 +3,11 @@ require 'rainbow/ext/string'
 module Geo
   class PromoteToPrimaryNode
-    def initialize(base_path, options)
-      @base_path = base_path
+    attr_accessor :ctl, :base_path
+
+    def initialize(ctl, options)
+      @ctl = ctl
+      @base_path = @ctl.base_path
       @options = options
     end
@@ -71,11 +74,7 @@ module Geo
     end
     def promote_postgresql_to_primary
-      puts
-      puts 'Promoting the PostgreSQL to primary...'.color(:yellow)
-      puts
-
-      run_command('/opt/gitlab/embedded/bin/gitlab-pg-ctl promote', live: true).error!
+      Geo::PromoteDb.new(ctl).execute
     end
     def reconfigure

files/gitlab-ctl-commands-ee/lib/geo/replication_toggle_command.rb

 require_relative "./replication_process"
+require_relative "./promote_db"
 require 'optparse'
 require 'English'
@@ -19,6 +20,8 @@ module Geo
     def execute!
       @replication_process.send(@action.to_sym)
+
+      process_pitr_file
     rescue Geo::PsqlError => e
       puts "Postgres encountered an error: #{e.message}"
       exit 1
@@ -33,6 +36,30 @@ module Geo
     private
+    attr_reader :action, :ctl
+
+    def process_pitr_file
+      geo_pitr_file_path = "#{ctl.data_path}/postgresql/data/#{Geo::PromoteDb::PITR_FILE_NAME}"
+
+      if action == 'pause'
+        puts "* Create Geo point-in-time recovery file".color(:green)
+
+        File.write(geo_pitr_file_path, current_lsn)
+      elsif action == 'resume'
+        puts "* Remove Geo point-in-time recovery file".color(:green)
+
+        File.delete(geo_pitr_file_path) if File.exist?(geo_pitr_file_path)
+      end
+    end
+
+    def current_lsn
+      run_query('SELECT pg_last_wal_replay_lsn()')
+    end
+
+    def run_query(query)
+      GitlabCtl::Util.get_command_output("gitlab-psql -d postgres -c '#{query}' -q -t").strip
+    end
+
     def parse_options!
       opts_parser = OptionParser.new do |opts|
         opts.banner = "Usage: gitlab-ctl replication-process-#{@action} [options]"

files/gitlab-ctl-commands-ee/promote_db.rb

+require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/geo/promote_db"
+
+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_command_under_category('promote-db', 'gitlab-geo', 'Promote secondary PostgreSQL database', 2) do |cmd_name, *args|
+  Geo::PromoteDb.new(self).execute
+end

files/gitlab-ctl-commands-ee/promote_to_primary_node.rb

@@ -40,5 +40,5 @@ add_command_under_category('promote-to-primary-node', 'gitlab-geo', 'Promote to
     options
   end
-  Geo::PromoteToPrimaryNode.new(base_path, get_ctl_options).execute
+  Geo::PromoteToPrimaryNode.new(self, get_ctl_options).execute
 end

spec/gitlab-ctl-commands-ee/lib/geo/promote_db_spec.rb

+require 'spec_helper'
+require 'geo/promote_db'
+require 'gitlab_ctl/util'
+
+RSpec.describe Geo::PromoteDb, '#execute' do
+  let(:instance) { double(base_path: '/opt/gitlab/embedded', data_path: '/var/opt/gitlab/postgresql/data') }
+
+  subject(:command) { described_class.new(instance) }
+
+  before do
+    allow($stdout).to receive(:puts)
+    allow($stdout).to receive(:print)
+
+    allow(command).to receive(:run_command).with(any_args)
+
+    allow(command).to receive(:run_command).and_return(double('error!' => nil))
+  end
+
+  context 'when PITR file does not exist' do
+    it 'does not run PITR recovery' do
+      expect(command).not_to receive(:write_recovery_settings)
+
+      command.execute
+    end
+  end
+
+  context 'when PITR file exists' do
+    let(:lsn) { '16/B374D848' }
+
+    before do
+      allow(command).to receive(:lsn_from_pitr_file).and_return(lsn)
+    end
+
+    it 'runs PITR recovery' do
+      expect(command).to receive(:write_recovery_settings).with(lsn)
+
+      expect { command.execute }.to output(
+        /Recovery to point #{lsn} and promoting.../).to_stdout
+    end
+
+    context 'PG version 11' do
+      it 'runs PITR recovery' do
+        allow(command).to receive(:postgresql_version).and_return(11)
+        expect(command).to receive(:write_recovery_conf)
+
+        expect { command.execute }.to output(
+          /Writing recovery.conf/).to_stdout
+      end
+    end
+
+    context 'PG version 12' do
+      it 'runs PITR recovery' do
+        allow(command).to receive(:postgresql_version).and_return(12)
+        expect(command).to receive(:write_geo_config_file)
+
+        expect { command.execute }.to output(
+          /PostgreSQL 12 or newer. Writing settings to postgresql.conf/).to_stdout
+      end
+    end
+  end
+end

spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node_spec.rb

 require 'spec_helper'
 require 'fileutils'
 require 'geo/promote_to_primary_node'
+require 'geo/promote_db'
 require 'geo/promotion_preflight_checks'
 require 'gitlab_ctl/util'
 RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
   let(:options) { { skip_preflight_checks: true } }
-  subject(:command) { described_class.new(nil, options) }
+  let(:instance) { double(base_path: '/opt/gitlab/embedded', data_path: '/var/opt/gitlab/postgresql/data') }
+
+  subject(:command) { described_class.new(instance, options) }
   let(:config_path) { Dir.mktmpdir }
   let(:gitlab_config_path) { File.join(config_path, 'gitlab.rb') }
@@ -23,6 +26,23 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
     FileUtils.rm_rf(config_path)
   end
+  describe '#promote_postgresql_to_primary' do
+    before do
+      allow(STDIN).to receive(:gets).and_return('y')
+
+      allow(command).to receive(:toggle_geo_roles).and_return(true)
+      allow(command).to receive(:reconfigure).and_return(true)
+      allow(command).to receive(:promote_to_primary).and_return(true)
+      allow(command).to receive(:success_message).and_return(true)
+    end
+
+    it 'promotes the database' do
+      expect_any_instance_of(Geo::PromoteDb).to receive(:execute)
+
+      command.execute
+    end
+  end
+
   describe '#run_preflight_checks' do
     before do
       allow(STDIN).to receive(:gets).and_return('y')
@@ -58,7 +78,7 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
       it 'passes given options to preflight checks command' do
         expect(Geo::PromotionPreflightChecks).to receive(:new).with(
-          nil, options).and_call_original
+          '/opt/gitlab/embedded', options).and_call_original
         command.execute
       end

spec/gitlab-ctl-commands-ee/lib/geo/replication_toggle_command_spec.rb

@@ -4,19 +4,26 @@ $LOAD_PATH << './files/gitlab-ctl-commands-ee/lib'
 $LOAD_PATH << './files/gitlab-ctl-commands/lib'
 require 'geo/replication_toggle_command'
+require 'geo/promote_to_primary_node'
+require 'gitlab_ctl/util'
 RSpec.describe Geo::ReplicationToggleCommand do
   let(:status) { double('Command status', error?: false) }
   let(:arguments) { [] }
-  let(:ctl_instance) { double('gitlab-ctl instance', base_path: '') }
+  let(:ctl_instance) { double('gitlab-ctl instance', base_path: '', data_path: 'data_path') }
+
+  before do
+    allow_any_instance_of(Geo::ReplicationToggleCommand).to receive(:current_lsn).and_return('16/B374D848')
+  end
   describe 'pause' do
     subject { described_class.new(ctl_instance, 'pause', arguments) }
     it 'calls pause' do
       expect_any_instance_of(Geo::ReplicationProcess).to receive(:pause)
-      subject.execute!
+      expect(File).to receive(:write).with('data_path/postgresql/data/geo-pitr-file', '16/B374D848')
+      expect { subject.execute! }.to output(/Create Geo point-in-time recovery file/).to_stdout
     end
     it 'rescues and exits if postgres has an error' do
@@ -33,8 +40,9 @@ RSpec.describe Geo::ReplicationToggleCommand do
       it 'uses the specified database' do
         expect(Geo::ReplicationProcess).to receive(:new).with(any_args, { db_name: 'database_i_want' }).and_call_original
         expect_any_instance_of(Geo::ReplicationProcess).to receive(:pause)
-        subject.execute!
+        expect(File).to receive(:write).with('data_path/postgresql/data/geo-pitr-file', '16/B374D848')
+        expect { subject.execute! }.to output(/Create Geo point-in-time recovery file/).to_stdout
       end
     end
   end
@@ -45,7 +53,7 @@ RSpec.describe Geo::ReplicationToggleCommand do
     it 'calls resume' do
       expect_any_instance_of(Geo::ReplicationProcess).to receive(:resume)
-      subject.execute!
+      expect { subject.execute! }.to output(/Remove Geo point-in-time recovery file/).to_stdout
     end
     it 'rescues and exits if postgres has an error' do
@@ -71,7 +79,7 @@ RSpec.describe Geo::ReplicationToggleCommand do
         expect(Geo::ReplicationProcess).to receive(:new).with(any_args, { db_name: 'database_i_want' }).and_call_original
         expect_any_instance_of(Geo::ReplicationProcess).to receive(:resume)
-        subject.execute!
+        expect { subject.execute! }.to output(/Remove Geo point-in-time recovery file/).to_stdout
       end
     end
   end

spec/gitlab-ctl-commands-ee/promote_db_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'geo/promote_db'
+
+RSpec.describe 'gitlab-ctl promote-db' do
+  let(:klass) { Geo::PromoteDb }
+  let(:command_name) { 'promote-db' }
+  let(:command_script) { 'promote_db' }
+
+  include_context 'ctl'
+
+  it_behaves_like 'gitlab geo promotion commands', 'promote-db'
+end