Check if primary can be contacted

changelogs/unreleased/ag-check-if-primary-can-be-contacted.yml

+---
+title: Geo - Confirm if primary can be contacted after manual preflight checks
+merge_request: 4260
+author:
+type: changed

files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node.rb

@@ -11,8 +11,6 @@ module Geo
     def execute
       run_preflight_checks
-      make_sure_primary_is_down
-
       promote_postgresql_to_primary
       reconfigure
@@ -27,23 +25,7 @@ module Geo
     def run_preflight_checks
       return true if @options[:skip_preflight_checks]
-      PromotionPreflightChecks.new.execute
-    end
-
-    def make_sure_primary_is_down
-      return true if @options[:confirm_primary_is_down]
-
-      puts
-      puts '---------------------------------------'.color(:yellow)
-      puts 'WARNING: Make sure your primary is down'.color(:yellow)
-      puts 'If you have more than one secondary please see https://docs.gitlab.com/ee/gitlab-geo/disaster-recovery.html#promoting-secondary-geo-replica-in-multi-secondary-configurations'.color(:yellow)
-      puts 'There may be data saved to the primary that was not been replicated to the secondary before the primary went offline. This data should be treated as lost if you proceed.'.color(:yellow)
-      puts '---------------------------------------'.color(:yellow)
-      puts
-
-      print '*** Are you sure? (N/y): '.color(:green)
-
-      raise 'Exited because primary node must be down' unless STDIN.gets.chomp.casecmp('y').zero?
+      PromotionPreflightChecks.new(@base_path, @options).execute
     end
     def promote_postgresql_to_primary
@@ -51,7 +33,7 @@ module Geo
       puts 'Promoting the PostgreSQL to primary...'.color(:yellow)
       puts
-      run_command("/opt/gitlab/embedded/bin/gitlab-pg-ctl promote", live: true).error!
+      run_command('/opt/gitlab/embedded/bin/gitlab-pg-ctl promote', live: true).error!
     end
     def reconfigure

files/gitlab-ctl-commands-ee/lib/geo/promotion_preflight_checks.rb

@@ -3,11 +3,21 @@ require 'rainbow/ext/string'
 module Geo
   class PromotionPreflightChecks
+    def initialize(base_path, options)
+      @base_path = base_path
+      @options = options
+    end
+
     def execute
-      confirm_manual_checks
-    rescue RuntimeError => e
-      puts e.message
-      exit 1
+      begin
+        confirm_manual_checks
+        confirm_primary_is_down
+      rescue RuntimeError => e
+        puts e.message
+        exit 1
+      end
+
+      success_message
     end
     private
@@ -34,5 +44,32 @@ module Geo
         'Please read https://docs.gitlab.com/ee/administration/geo/'\
         'disaster_recovery/planned_failover.html#preflight-checks'.color(:red)
     end
+
+    def confirm_primary_is_down
+      return true if @options[:confirm_primary_is_down]
+
+      puts
+      puts '---------------------------------------'.color(:yellow)
+      puts 'WARNING: Make sure your primary is down'.color(:yellow)
+      puts 'If you have more than one secondary please see '\
+        'https://docs.gitlab.com/ee/gitlab-geo/disaster-recovery.html#'\
+        'promoting-secondary-geo-replica-in-multi-secondary-configurations'.color(:yellow)
+      puts 'There may be data saved to the primary that was not been '\
+        'replicated to the secondary before the primary went offline. '\
+        'This data should be treated as lost if you proceed.'.color(:yellow)
+      puts '---------------------------------------'.color(:yellow)
+      puts
+
+      print '*** Is primary down? (N/y): '.color(:green)
+
+      return if STDIN.gets.chomp.casecmp('y').zero?
+
+      raise 'ERROR: Primary node must be down.'.color(:red)
+    end
+
+    def success_message
+      puts
+      puts 'All preflight checks have passed. This node can now be promoted.'.color(:green)
+    end
   end
 end

files/gitlab-ctl-commands-ee/promotion_preflight_checks.rb

 require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/geo/promotion_preflight_checks"
-add_command_under_category('promotion-preflight-checks', 'gitlab-geo', 'Run preflight checks for promotion to primary node', 2) do |cmd_name|
-  Geo::PromotionPreflightChecks.new.execute
+add_command_under_category('promotion-preflight-checks', 'gitlab-geo', 'Run preflight checks for promotion to primary node', 2) do |cmd_name, *args|
+  Geo::PromotionPreflightChecks.new(base_path, get_ctl_options).execute
+end
+
+def get_ctl_options
+  options = {}
+  OptionParser.new do |opts|
+    opts.banner = "Usage: gitlab-ctl promotion-preflight-checks [options]"
+
+    opts.on('-p', '--[no-]confirm-primary-is-down', 'Do not ask for confirmation that primary is down') do |p|
+      options[:confirm_primary_is_down] = p
+    end
+  end.parse!(ARGV.dup)
+
+  options
 end

spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node_spec.rb

@@ -21,6 +21,14 @@ describe Geo::PromoteToPrimaryNode, '#execute' do
     FileUtils.rm_rf(temp_directory)
   end
+  shared_examples 'runs promotion preflight checks' do |expected_args|
+    it do
+      expect_any_instance_of(Geo::PromotionPreflightChecks).to receive(:execute)
+
+      command.execute
+    end
+  end
+
   describe '#run_preflight_checks' do
     before do
       allow(STDIN).to receive(:gets).and_return('y')
@@ -32,24 +40,37 @@ describe Geo::PromoteToPrimaryNode, '#execute' do
     context 'when `--skip-preflight-checks` is passed' do
       it 'does not run execute promotion preflight checks' do
-        expect(Geo::PromotionPreflightChecks).not_to receive(:execute)
+        expect_any_instance_of(Geo::PromotionPreflightChecks).not_to receive(:execute)
         command.execute
       end
     end
     context 'when `--skip-preflight-checks` is not passed' do
-      let(:options) { {} }
-      it 'runs promotion preflight checks' do
-        expect_any_instance_of(Geo::PromotionPreflightChecks).to receive(:execute)
-        command.execute
+      context 'when --confirm-primary-is-down is not passed' do
+        let(:options) { {} }
+        it_behaves_like 'runs promotion preflight checks',
+                        '--no-confirm-primary-is-down'
+      end
+      context 'when --no-confirm-primary-is-down is passed' do
+        let(:options) { { confirm_primary_is_down: false } }
+
+        it_behaves_like 'runs promotion preflight checks',
+                        '--no-confirm-primary-is-down'
+      end
+
+      context 'when --confirm-primary-is-down is passed' do
+        let(:options) { { confirm_primary_is_down: true } }
+
+        it_behaves_like 'runs promotion preflight checks',
+                        '--confirm-primary-is-down'
       end
     end
   end
-  context 'when confirmation is accepted' do
+  context 'when preflight checks pass' do
     before do
       allow(STDIN).to receive(:gets).and_return('y')
     end
@@ -65,15 +86,13 @@ describe Geo::PromoteToPrimaryNode, '#execute' do
     end
   end
-  context 'when confirmation is refused' do
+  context 'when preflight checks fail' do
     before do
       allow(STDIN).to receive(:gets).and_return('n')
     end
-    it 'calls all the subcommands' do
-      is_expected.not_to receive(:run_command)
-
-      expect { command.execute }.to raise_error RuntimeError, 'Exited because primary node must be down'
+    it 'raises an error' do
+      expect { command.execute }.to raise_error
     end
   end
 end

spec/gitlab-ctl-commands-ee/lib/geo/promotion_preflight_checks_spec.rb

@@ -4,9 +4,10 @@ require 'geo/promotion_preflight_checks'
 require 'gitlab_ctl/util'
 describe Geo::PromotionPreflightChecks, '#execute' do
-  subject(:command) { described_class.new }
-
   let(:confirmation) { 'y' }
+  let(:options) { { confirm_primary_is_down: true } }
+
+  subject(:command) { described_class.new(nil, options) }
   before do
     allow(STDIN).to receive(:gets).and_return(confirmation)
@@ -18,13 +19,13 @@ describe Geo::PromotionPreflightChecks, '#execute' do
       .to_stdout
   end
-  context 'when confirmation is accepted' do
+  context 'when manual checks are confirmed' do
     it 'does not raise an error' do
       expect { command.execute }.to_not raise_error
     end
   end
-  context 'when confirmation is not accepted' do
+  context 'when manual checks are not confirmed' do
     let(:confirmation) { 'n' }
     around do |example|
@@ -38,4 +39,53 @@ describe Geo::PromotionPreflightChecks, '#execute' do
       ).to_stdout
     end
   end
+
+  describe 'option --confirm-primary-is-down' do
+    before do
+      allow(command).to receive(:confirm_manual_checks).and_return(true)
+    end
+
+    context 'when the option is not passed' do
+      let(:options) { {} }
+      let(:confirmation) { 'n' }
+
+      around do |example|
+        example.run
+      rescue SystemExit
+      end
+
+      it 'asks user for confirmation' do
+        expect { command.execute }.to output(
+          /Is primary down? (N\/y)/)
+          .to_stdout
+      end
+
+      it 'prints an error message when user doesn not select y/Y' do
+        expect { command.execute }.to output(
+          /ERROR: Primary node must be down./)
+          .to_stdout
+      end
+    end
+
+    context 'when the option is passed' do
+      it 'does not ask user for confirmation' do
+        expect { command.execute }.not_to output(
+          /Is primary down? (N\/y)/)
+          .to_stdout
+      end
+    end
+  end
+
+  context 'when all checks pass' do
+    before do
+      allow(command).to receive(:confirm_manual_checks).and_return(true)
+      allow(command).to receive(:confirm_primary_is_down).and_return(true)
+    end
+
+    it 'prints a success message' do
+      expect { command.execute }.to output(
+        /All preflight checks have passed. This node can now be promoted./)
+        .to_stdout
+    end
+  end
 end

spec/gitlab-ctl-commands-ee/promotion_preflight_checks_spec.rb

@@ -12,4 +12,8 @@ describe 'gitlab-ctl promotion-preflight-checks' do
   include_context 'ctl'
   it_behaves_like 'gitlab geo promotion commands'
+
+  it_behaves_like 'geo promotion command accepts option',
+                  '--confirm-primary-is-down',
+                  { confirm_primary_is_down: true }
 end