Allow promotion task to alter Geo node configuration

This is a proof-of-concept that adds a separate
configuration  for the GitLab cluster with the
more "high-level settings" for a Geo setup and
allows the Geo promotion task to alter these
settings.

changelogs/unreleased/5384-poc-allow-promotion-task-to-alter-geo-node-configuration.yml

+---
+title: Allow promotion task to alter Geo node configuration
+merge_request: 4609
+author:
+type: added

files/gitlab-cookbooks/package/libraries/helpers/gitlab_cluster_helper.rb

+require 'chef/json_compat'
+require 'chef/log'
+require_relative 'logging_helper'
+
+class GitlabClusterHelper
+  CONFIG_PATH = '/etc/gitlab'.freeze
+  JSON_FILE   = '/etc/gitlab/gitlab-cluster.json'.freeze
+
+  class << self
+    def config_available?
+      File.exist?(JSON_FILE)
+    end
+  end
+
+  def config
+    return @config if defined?(@config)
+
+    @config = load_from_file
+  end
+
+  # Roles defined in the JSON file overrides roles from /etc/gitlab/gitlab.rb
+  def load_roles!
+    load_role!('geo_primary_role', 'primary')
+    load_role!('geo_secondary_role', 'secondary')
+  end
+
+  # Write configuration to the local JSON file overriding current settings
+  def write_to_file!
+    return unless File.directory?(CONFIG_PATH)
+
+    json_config = Chef::JSONCompat.to_json_pretty(config)
+
+    File.open(JSON_FILE, 'w', 0600) do |f|
+      f.puts(json_config)
+      f.chmod(0600)
+    end
+  end
+
+  private
+
+  # Load configuration from the local JSON file
+  def load_from_file
+    return {} unless self.class.config_available?
+
+    Chef::JSONCompat.from_json(File.read(JSON_FILE))
+  end
+
+  def load_role!(role, key)
+    return unless config.key?(key)
+
+    print_warning(role, key) if Gitlab[role]['enable']
+    Gitlab[role]['enable'] = config[key]
+  end
+
+  def print_warning(role, key)
+    LoggingHelper.warning "The #{role} is defined in #{JSON_FILE} as #{key} and takes priority over the role in the /etc/gitlab/gitlab.rb"
+  end
+end

files/gitlab-cookbooks/package/libraries/helpers/settings_helper.rb

@@ -22,6 +22,7 @@ require 'securerandom'
 require 'uri'
 require_relative '../config_mash.rb'
+require_relative 'gitlab_cluster_helper'
 module SettingsHelper
   def self.extended(base)
@@ -153,6 +154,9 @@ module SettingsHelper
     Services.enable_group(Services::SYSTEM_GROUP)
     RolesHelper.parse_enabled
+    # Roles defined in the cluster configuration file overrides roles from /etc/gitlab/gitlab.rb
+    gitlab_cluster_helper.load_roles!
+
     # Load our roles
     DefaultRole.load_role
     @available_roles.each do |key, value|
@@ -203,6 +207,10 @@ module SettingsHelper
   private
+  def gitlab_cluster_helper
+    @gitlab_cluster_helper ||= GitlabClusterHelper.new
+  end
+
   # Sort settings by their sequence value
   def sorted_settings
     @settings.select { |_k, value| !value[:ee] || Gitlab['edition'] == :ee }.sort_by { |_k, value| value[:priority] }

files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node.rb

@@ -10,15 +10,11 @@ module Geo
     def execute
       run_preflight_checks
-
       final_confirmation
-
+      toggle_geo_roles
       promote_postgresql_to_primary
-
       reconfigure
-
       promote_to_primary
-
       success_message
     end
@@ -60,6 +56,20 @@ module Geo
       exit 1
     end
+    def toggle_geo_roles
+      puts
+      puts 'Disabling the secondary role and enabling the primary in the cluster configuration file...'.color(:yellow)
+      puts
+
+      cluster_helper = GitlabClusterHelper.new
+      cluster_helper.config['primary'] = true
+      cluster_helper.config['secondary'] = false
+      return if cluster_helper.write_to_file!
+
+      puts "ERROR: Could not write to #{GitlabClusterHelper::JSON_FILE}.".color(:red)
+      exit 1
+    end
+
     def promote_postgresql_to_primary
       puts
       puts 'Promoting the PostgreSQL to primary...'.color(:yellow)

files/gitlab-ctl-commands-ee/promote_to_primary_node.rb

+require "#{base_path}/embedded/cookbooks/package/libraries/helpers/gitlab_cluster_helper"
 require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/geo/promote_to_primary_node"
 #

spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node_spec.rb

@@ -9,8 +9,8 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
   subject(:command) { described_class.new(nil, options) }
-  let(:temp_directory) { Dir.mktmpdir }
-  let(:gitlab_config_path) { File.join(temp_directory, 'gitlab.rb') }
+  let(:config_path) { Dir.mktmpdir }
+  let(:gitlab_config_path) { File.join(config_path, 'gitlab.rb') }
   before do
     allow($stdout).to receive(:puts)
@@ -20,13 +20,14 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
   end
   after do
-    FileUtils.rm_rf(temp_directory)
+    FileUtils.rm_rf(config_path)
   end
   describe '#run_preflight_checks' do
     before do
       allow(STDIN).to receive(:gets).and_return('y')
+      allow(command).to receive(:toggle_geo_roles).and_return(true)
       allow(command).to receive(:promote_postgresql_to_primary).and_return(true)
       allow(command).to receive(:reconfigure).and_return(true)
       allow(command).to receive(:promote_to_primary).and_return(true)
@@ -64,6 +65,53 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
     end
   end
+  describe '#toggle_geo_roles' do
+    let(:gitlab_cluster_config_path) { File.join(config_path, 'gitlab-cluster.json') }
+
+    before do
+      stub_const('GitlabClusterHelper::CONFIG_PATH', config_path)
+      stub_const('GitlabClusterHelper::JSON_FILE', gitlab_cluster_config_path)
+
+      allow(STDIN).to receive(:gets).and_return('y')
+
+      allow(command).to receive(:run_preflight_checks).and_return(true)
+      allow(command).to receive(:promote_postgresql_to_primary).and_return(true)
+      allow(command).to receive(:reconfigure).and_return(true)
+      allow(command).to receive(:promote_to_primary).and_return(true)
+      allow(command).to receive(:success_message).and_return(true)
+    end
+
+    context 'when the cluster configuration file does not exist' do
+      it 'creates the file with the Geo primary role enabled and secondary role disabled' do
+        command.execute
+
+        expect(File.exist?(gitlab_cluster_config_path)).to eq(true)
+        expect(read_file_content(gitlab_cluster_config_path)).to eq("primary" => true, "secondary" => false)
+      end
+    end
+
+    context 'when the cluster configuration file exists' do
+      it 'disables the Geo secondary role' do
+        write_file_content(gitlab_cluster_config_path, primary: false, secondary: true)
+
+        command.execute
+
+        expect(read_file_content(gitlab_cluster_config_path)).to eq("primary" => true, "secondary" => false)
+      end
+    end
+
+    def read_file_content(fullpath)
+      JSON.parse(File.read(fullpath))
+    end
+
+    def write_file_content(fullpath, content)
+      File.open(fullpath, 'w') do |f|
+        f.write(content.to_json)
+        f.chmod(0600)
+      end
+    end
+  end
+
   context 'when preflight checks pass' do
     before do
       allow(STDIN).to receive(:gets).and_return('y')
@@ -71,6 +119,7 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
       allow_any_instance_of(Geo::PromotionPreflightChecks).to receive(
         :execute).and_return(true)
+      allow(command).to receive(:toggle_geo_roles).and_return(true)
       allow(command).to receive(:promote_postgresql_to_primary).and_return(true)
       allow(command).to receive(:reconfigure).and_return(true)
       allow(command).to receive(:promote_to_primary).and_return(true)
@@ -95,8 +144,12 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
       end
       context 'when final confirmation is given' do
-        it 'calls the next subcommand' do
+        it 'calls all the subcommands' do
+          expect(command).to receive(:toggle_geo_roles)
           expect(command).to receive(:promote_postgresql_to_primary)
+          expect(command).to receive(:reconfigure)
+          expect(command).to receive(:promote_to_primary)
+          expect(command).to receive(:success_message)
           command.execute
         end
@@ -141,9 +194,11 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
       it 'calls all the subcommands if user affirms' do
         allow(STDIN).to receive(:gets).and_return('y')
+        is_expected.to receive(:toggle_geo_roles)
         is_expected.to receive(:promote_postgresql_to_primary)
         is_expected.to receive(:reconfigure)
         is_expected.to receive(:promote_to_primary)
+        is_expected.to receive(:success_message)
         command.execute
       end
@@ -157,4 +212,47 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
       end
     end
   end
+
+  context 'when writing to the cluster configuration file fail' do
+    around do |example|
+      example.run
+    rescue SystemExit
+    end
+
+    before do
+      allow(STDIN).to receive(:gets).and_return('y')
+
+      allow(command).to receive(:run_preflight_checks).and_return(true)
+
+      allow_any_instance_of(GitlabClusterHelper)
+        .to receive(:write_to_file!).and_return(false)
+    end
+
+    it 'exits with 1' do
+      expect { command.execute }.to raise_error(SystemExit)
+    end
+  end
+
+  context 'when writing to the cluster configuration file succeed' do
+    before do
+      allow(STDIN).to receive(:gets).and_return('y')
+
+      allow(command).to receive(:promote_postgresql_to_primary).and_return(true)
+      allow(command).to receive(:reconfigure).and_return(true)
+      allow(command).to receive(:promote_to_primary).and_return(true)
+      allow(command).to receive(:success_message).and_return(true)
+
+      allow_any_instance_of(GitlabClusterHelper)
+        .to receive(:write_to_file!).and_return(true)
+    end
+
+    it 'calls all the subcommands' do
+      expect(command).to receive(:promote_postgresql_to_primary)
+      expect(command).to receive(:reconfigure)
+      expect(command).to receive(:promote_to_primary)
+      expect(command).to receive(:success_message)
+
+      command.execute
+    end
+  end
 end

spec/libraries/helpers/gitlab_cluster_helper_spec.rb

+require 'spec_helper'
+require_relative '../../../files/gitlab-cookbooks/package/libraries/helpers/gitlab_cluster_helper'
+
+RSpec.describe GitlabClusterHelper do
+  let(:gitlab_cluster_config_path) { described_class::JSON_FILE }
+
+  describe '.config_available?' do
+    context 'when cluster configuration file exists' do
+      it 'returns true' do
+        allow(File).to receive(:exist?).with(gitlab_cluster_config_path).and_return(true)
+
+        expect(described_class.config_available?).to eq(true)
+      end
+    end
+
+    context 'when cluster configuration file does not exist' do
+      it 'returns false' do
+        expect(described_class.config_available?).to eq(false)
+      end
+    end
+  end
+
+  describe '#config' do
+    context 'when the cluster configuration file does not exist' do
+      it 'returns an empty hash' do
+        expect(subject.config).to be_empty
+      end
+    end
+
+    context 'when the cluster configuration file exists' do
+      it 'parses the file content' do
+        stub_file_content(gitlab_cluster_config_path, foo: 'bar')
+
+        expect(subject.config).to eq('foo' => 'bar')
+      end
+    end
+  end
+
+  describe '#load_roles!' do
+    before do
+      stub_gitlab_rb(application_role: { enable: true }, geo_primary_role: { enable: nil }, geo_secondary_role: { enable: true })
+    end
+
+    it 'overrides roles defined in the configuration file' do
+      stub_file_content(gitlab_cluster_config_path, secondary: false)
+
+      subject.load_roles!
+
+      expect(Gitlab['application_role']['enable']).to eq(true)
+      expect(Gitlab['geo_secondary_role']['enable']).to eq(false)
+    end
+
+    it 'does not override roles not defined in the configuration file' do
+      stub_file_content(gitlab_cluster_config_path, {})
+
+      subject.load_roles!
+
+      expect(Gitlab['application_role']['enable']).to eq(true)
+      expect(Gitlab['geo_secondary_role']['enable']).to eq(true)
+    end
+
+    it 'prints a warning message for each enabled role defined in the configuration file' do
+      stub_file_content(gitlab_cluster_config_path, primary: true, secondary: false)
+
+      expect(LoggingHelper)
+        .not_to receive(:warning)
+        .with("The geo_primary_role is defined in #{gitlab_cluster_config_path} as primary and takes priority over the role in the /etc/gitlab/gitlab.rb")
+
+      expect(LoggingHelper)
+        .to receive(:warning)
+        .with("The geo_secondary_role is defined in #{gitlab_cluster_config_path} as secondary and takes priority over the role in the /etc/gitlab/gitlab.rb")
+        .once
+
+      subject.load_roles!
+    end
+  end
+
+  describe '#write_to_file!' do
+    let(:config_path) { Dir.mktmpdir }
+    let(:gitlab_cluster_config_path) { File.join(config_path, 'gitlab-cluster.json') }
+
+    before do
+      stub_const('GitlabClusterHelper::CONFIG_PATH', config_path)
+      stub_const('GitlabClusterHelper::JSON_FILE', gitlab_cluster_config_path)
+    end
+
+    after do
+      FileUtils.rm_rf(config_path)
+    end
+
+    context 'when the config directory does not exist' do
+      it 'does not create the configuration file' do
+        FileUtils.rm_rf(config_path)
+
+        subject.write_to_file!
+
+        expect(File.exist?(gitlab_cluster_config_path)).to eq(false)
+      end
+    end
+
+    context 'when the cluster configuration file does not exist' do
+      it 'creates the configuration file' do
+        FileUtils.rm_rf(gitlab_cluster_config_path)
+
+        subject.write_to_file!
+
+        expect(File.exist?(gitlab_cluster_config_path)).to eq(true)
+        expect(read_file_content(gitlab_cluster_config_path)).to be_empty
+      end
+    end
+
+    context 'when the cluster configuration file exists' do
+      it 'overrides previous settings' do
+        write_file_content(gitlab_cluster_config_path, foo: 'bar', zoo: true)
+        subject.config['zoo'] = false
+
+        subject.write_to_file!
+
+        expect(read_file_content(gitlab_cluster_config_path)).to eq("foo" => "bar", "zoo" => false)
+      end
+    end
+  end
+
+  def stub_file_content(fullpath, content)
+    allow(File).to receive(:exist?).with(fullpath).and_return(true)
+    allow(IO).to receive(:read).with(fullpath).and_return(content.to_json)
+  end
+
+  def read_file_content(fullpath)
+    JSON.parse(File.read(fullpath))
+  end
+
+  def write_file_content(fullpath, content)
+    File.open(fullpath, 'w') do |f|
+      f.write(content.to_json)
+      f.chmod(0600)
+    end
+  end
+end

spec/support/shared_examples/ctl_shared_context.rb

@@ -3,8 +3,16 @@ require 'omnibus-ctl'
 RSpec.shared_context 'ctl' do
   let(:ctl) { Omnibus::Ctl.new('testing-ctl') }
+
   before do
     allow_any_instance_of(Omnibus::Ctl).to receive(:require).and_call_original
+
+    allow_any_instance_of(Omnibus::Ctl).to receive(:require).with(
+      "/opt/testing-ctl/embedded/cookbooks/package/libraries/helpers/gitlab_cluster_helper"
+    ) do
+      require_relative("../../../files/gitlab-cookbooks/package/libraries/helpers/gitlab_cluster_helper")
+    end
+
     allow_any_instance_of(Omnibus::Ctl).to receive(:require).with(
       "/opt/testing-ctl/embedded/service/omnibus-ctl-ee/lib/geo/#{command_script}"
     ) do