Geo: Add a command to promote secondary node to primary

CHANGELOG.md

@@ -15,6 +15,7 @@ omnibus-gitlab repository.
 - Enable support for git push options in the git config (Romain Maffina) 08edd3e4
 - Update Redis to 3.2.11 (Takuya Noguchi)
 - Turn on postgresql SSL by default b18597e3
+- [GitLab Geo] Add a command to promote secondary node to primary
 10.2.3

files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary.rb

+require 'io/console'
+require 'rainbow/ext/string'
+
+module Geo
+  class PromoteToPrimary
+    TRIGGER_FILE_PATH = '/tmp/postgresql.trigger'.freeze
+
+    def initialize(base_path, options)
+      @base_path = base_path
+      @options = options
+    end
+
+    def execute
+      make_sure_primary_is_down
+
+      promote_postgresql_to_primary
+
+      remove_ssh_keys
+
+      reconfigure
+
+      promote_to_primary
+
+      notification_about_rsync
+    end
+
+    private
+
+    def git_user_home
+      GitlabCtl::Util.get_node_attributes(@base_path)['gitlab']['user']['home']
+    end
+
+    def make_sure_primary_is_down
+      return true if @options[:confirm_primary_is_down]
+
+      puts
+      puts '---------------------------------------'.color(:yellow)
+      puts 'WARNING: Make sure your primary is down and also be aware that'.color(:yellow)
+      puts 'this command only works for setups with one secondary.'.color(:yellow)
+      puts 'If you have more of them please follow documentation in https://docs.gitlab.com/ee/gitlab-geo/disaster-recovery.html'.color(:yellow)
+      puts '---------------------------------------'.color(:yellow)
+      puts
+
+      print '*** Are you sure? (N/y): '.color(:green)
+
+      unless STDIN.gets.chomp.downcase == 'y'
+        raise 'Exited because primary node must be down'
+      end
+    end
+
+    def promote_postgresql_to_primary
+      puts
+      puts 'Promoting the Postgres to primary...'.color(:yellow)
+      puts
+
+      run_command("touch #{TRIGGER_FILE_PATH}")
+    end
+
+    def remove_ssh_keys
+      return nil unless File.exist?(key_path) || File.exist?(public_key_path)
+
+      unless @options[:confirm_removing_keys]
+        puts
+        puts 'SSH keys detected! Remove? See https://docs.gitlab.com/ee/gitlab-geo/disaster-recovery.html#promoting-a-secondary-node for more information [Y/n]'.color(:yellow)
+
+        if STDIN.gets.chomp.downcase == 'n'
+          return true
+        end
+      end
+
+      [key_path, public_key_path].each do |path|
+        File.delete(path) if File.exist?(path)
+      end
+    end
+
+    def reconfigure
+      puts
+      puts 'Reconfiguring...'.color(:yellow)
+      puts
+
+      run_command('gitlab-ctl reconfigure')
+    end
+
+    def promote_to_primary
+      puts
+      puts 'Running gitlab-rake geo:set_secondary_as_primary...'.color(:yellow)
+      puts
+      run_command('gitlab-rake geo:set_secondary_as_primary')
+    end
+
+    def notification_about_rsync
+      puts
+      puts '---------------------------------------'.color(:yellow)
+      puts 'Note: Rsync everything in /var/opt/gitlab/gitlab-rails/uploads and /var/opt/gitlab/gitlab-rails/shared from your old node to the new one !!!'.color(:yellow)
+      puts '---------------------------------------'.color(:yellow)
+      puts
+    end
+
+    def run_command(cmd)
+      GitlabCtl::Util.run_command(cmd)
+    end
+
+    def key_path
+      @key_path ||= File.join(git_user_home, '.ssh/id_rsa')
+    end
+
+    def public_key_path
+      @public_key_path ||= File.join(git_user_home ,'.ssh/id_rsa.pub')
+    end
+  end
+end

files/gitlab-ctl-commands-ee/promote_to_primary_node.rb

+require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/geo/promote_to_primary"
+
+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_command_under_category('promote-to-primary-node', 'gitlab-geo', 'Promote to primary node', 2) do |cmd_name, *args|
+  Geo::PromoteToPrimary.new(base_path, get_ctl_options).execute
+end
+
+def get_ctl_options
+  options = {}
+  OptionParser.new do |opts|
+    opts.banner = "Usage: gitlab-ctl promote-to-primary-node [options]"
+
+    opts.on('-p', '--[no-]confirm-primary-is-down', 'Do not ask a confirmation that primary is down') do |p|
+      options[:confirm_primary_is_down] = p
+    end
+
+    opts.on('-c', '--[no-]confirm-removing-keys', 'Do not ask a confirmation about removing keys') do |c|
+      options[:confirm_removing_keys] = c
+    end
+  end.parse!(ARGV.dup)
+
+  options
+end

spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_spec.rb

+$LOAD_PATH << './files/gitlab-ctl-commands-ee/lib'
+$LOAD_PATH << './files/gitlab-ctl-commands/lib'
+
+require 'fileutils'
+require 'geo/promote_to_primary'
+require 'fileutils'
+require 'gitlab_ctl/util'
+
+describe Geo::PromoteToPrimary, '#execute' do
+  subject(:command) { described_class.new(nil, {}) }
+
+  let(:temp_directory) { Dir.mktmpdir }
+  let(:postgres_trigger_file_path) { File.join(temp_directory, 'test_trigger') }
+  let(:gitlab_config_path) { File.join(temp_directory, 'gitlab.rb') }
+  let(:key_path) { File.join(temp_directory, 'id_rsa') }
+  let(:public_key_path) { File.join(temp_directory, 'id_rsa.pub') }
+
+  before do
+    allow(STDIN).to receive(:gets).and_return('y')
+    allow(command).to receive(:puts)
+    allow(command).to receive(:print)
+  end
+
+  after do
+    FileUtils.rm_rf(temp_directory)
+  end
+
+  it 'calls all the subcommands' do
+    stub_env
+
+    is_expected.to receive(:run_command).exactly(3).times
+
+    command.execute
+  end
+
+  it 'applies all the changes' do
+    stub_env
+
+    allow(command).to receive(:run_command) do |cmd|
+      fake_run_command(cmd)
+    end
+
+    command.execute
+
+    expect(@reconfigure_has_been_run).to be_truthy
+    expect(@rake_task_has_been_run).to be_truthy
+    expect(File.exist?(postgres_trigger_file_path)).to be_truthy
+    expect(File.exist?(key_path)).to be_falsey
+    expect(File.exist?(public_key_path)).to be_falsey
+  end
+
+  def stub_env
+    FileUtils.rm_f(postgres_trigger_file_path)
+    stub_const("Geo::PromoteToPrimary::TRIGGER_FILE_PATH", postgres_trigger_file_path)
+    allow(subject).to receive(:key_path).and_return(key_path)
+    allow(subject).to receive(:public_key_path).and_return(public_key_path)
+  end
+
+  def fake_run_command(cmd)
+    if cmd == 'gitlab-ctl reconfigure'
+      @reconfigure_has_been_run = true
+      return
+    end
+
+    if cmd == 'gitlab-rake geo:set_secondary_as_primary'
+      @rake_task_has_been_run = true
+      return
+    end
+
+    GitlabCtl::Util.run_command(cmd)
+  end
+end

spec/gitlab-ctl-commands-ee/promote_to_primary_node_spec.rb

+require 'omnibus-ctl'
+require 'chef_helper'
+
+describe 'gitlab-ctl promote-to-primary-node' do
+  subject(:ctl) { Omnibus::Ctl.new('testing-ctl') }
+
+  before do
+    allow_any_instance_of(Omnibus::Ctl).to receive(:require).and_call_original
+    allow_any_instance_of(Omnibus::Ctl).to receive(:require).with(
+      '/opt/testing-ctl/embedded/service/omnibus-ctl-ee/lib/geo/promote_to_primary'
+    ) do
+      require_relative('../../files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary')
+    end
+
+    ctl.load_file('files/gitlab-ctl-commands-ee/promote_to_primary_node.rb')
+  end
+
+  it 'appends a geo replication command' do
+    expect(subject.get_all_commands_hash).to include('promote-to-primary-node')
+  end
+
+  it 'executes the command when called' do
+    # ARGV contains the commands that were passed to rspec, which are
+    # invalid for the omnibus-ctl commands
+    oldargv = ARGV
+    ARGV = [] # rubocop:disable Style/MutableConstant
+
+    expect_any_instance_of(Geo::PromoteToPrimary).to receive(:execute)
+
+    ctl.promote_to_primary_node
+
+    ARGV = oldargv
+  end
+end