Merge branch '2497-automate-repmgr-configuration' into 'master'

Resolve "Automate repmgr configuration"

Closes #2497

See merge request !1664

CHANGELOG.md

@@ -15,6 +15,7 @@ omnibus-gitlab repository.
 - Set max_replication_slots to 1 by default for primary Geo instances
 - Set TZ environment variable for Gitaly
 - Add Performance Bar `performance_bar_allowed_group` configuration
+- Automate repmgr configuration
 9.3.5

config/software/gitlab-ctl-ee.rb

@@ -29,8 +29,10 @@ source path: File.expand_path(
 build do
   if ee
-    copy File.expand_path(
-      'files/gitlab-ctl-commands-ee/*.rb', Omnibus::Config.project_root
-    ), "#{install_dir}/embedded/service/omnibus-ctl/"
+    ['', 'lib/'].each do |dir|
+      copy File.expand_path(
+        "files/gitlab-ctl-commands-ee/#{dir}*.rb", Omnibus::Config.project_root
+      ), "#{install_dir}/embedded/service/omnibus-ctl/#{dir}"
+    end
   end
 end

files/gitlab-config-template/gitlab.rb.template

@@ -1494,7 +1494,7 @@ external_url 'GENERATED_EXTERNAL_URL'
 # geo_postgresql['ha'] = false
 # geo_postgresql['dir'] = '/var/opt/gitlab/geo-postgresql'
 # geo_postgresql['data_dir'] = '/var/opt/gitlab/geo-postgresql/data'
-#
+
 ################################################################################
 # Pgbouncer (EE only)
 # See [GitLab PgBouncer documentation](http://docs.gitlab.com/omnibus/settings/database.html#enabling-pgbouncer-ee-only)
@@ -1536,3 +1536,15 @@ external_url 'GENERATED_EXTERNAL_URL'
 # pgbouncer['auth_query'] = 'SELECT username, password FROM public.pg_shadow_lookup($1)'
 # postgresql['pgbouncer_user'] = nil
 # postgresql['pgbouncer_user_password'] = nil
+#
+
+################################################################################
+# Repmgr (EE only)
+################################################################################
+# repmgr['cluster'] = 'gitlab_cluster'
+# repmgr['database'] = 'gitlab_repmgr'
+# repmgr['host'] = nil
+# repmgr['node_number'] = nil
+# repmgr['port'] = 5432
+# repmgr['trust_auth_cidr_addresses'] = []
+# repmgr['user'] = 'gitlab_repmgr'

files/gitlab-cookbooks/gitlab-ee/metadata.rb

@@ -13,3 +13,4 @@ supports "centos"
 depends "runit"
 depends "gitlab"
 depends "package"
+depends 'repmgr'

files/gitlab-cookbooks/gitlab-ee/recipes/default.rb

@@ -30,6 +30,16 @@ include_recipe 'gitlab::default'
   end
 end
+%w(
+  repmgr
+).each do |service|
+  if node[service]['enable']
+    include_recipe "#{service}::enable"
+  else
+    include_recipe "#{service}::disable"
+  end
+end
+
 include_recipe 'gitlab-ee::ssh_keys'
 # Geo secondary

files/gitlab-cookbooks/gitlab/README.md

+# gitlab Cookbook
+
+Configures the different components needed for an Omnibus installation of GitLab
+
+## Resources
+
+### postgresql_user
+
+Creates a user account in the PostgreSQL instance
+
+#### properties
+
+* 'username': The name of the user to create. Required
+* 'password': The password of the account. Optional. This can be the plaintext password, or an md5 hash of the password. The hash can be generated by running `echo -n 'PASSWORD+USERNAME' | md5sum` and should be passed in the form `md5PASSWORD_HASH`.
+* 'options': An array of options to enable on the user. Optional. See the [PostgreSQL documentation](https://www.postgresql.org/docs/9.6/static/sql-createuser.html) for the available options
+* 'helper': The helper object for interacting with the running database. Default: PgHelper.new(node)
+
+#### example
+
+Create the user 'bar' without a password or any options
+
+```ruby
+postgresql_user 'bar'
+```
+
+Create the superuser 'bar' with an md5 encoded password
+```ruby
+postgresql_user 'bar' do
+  options %w(SUPERUSER)
+  password 'md5........'
+end
+```
+
+### postgresql_database
+
+Creates a database in the PostgreSQL instance
+
+#### properties
+
+* 'database': The name of the database to create. Required.
+* 'owner': The user account that will be the owner of the database. Default: `node['gitlab']['postgresql']['user']`
+* 'helper': The helper object for interacting with the running database. Default: PgHelper.new(node)
+* 'database_port': The port the database instance is listening on. Default: `node['gitlab']['postgresql']['port']`
+* 'database_socket': The directory that the socket file exists in. Default: `node['gitlab']['postgresql']['unix_socket_directory']`
+
+#### example
+
+Create a database called 'foo' owned by the default postgresql user
+
+```ruby
+postgresql_database 'foo'
+```
+
+Create a database called 'foo' owned by user 'bar'
+
+```ruby
+postgresql_database 'foo' do
+  owner 'bar'
+end
+```

files/gitlab-cookbooks/gitlab/libraries/gitlab.rb

@@ -94,6 +94,7 @@ module Gitlab
   geo_postgresql Mash.new
   prometheus_monitoring Mash.new
   pgbouncer Mash.new
+  repmgr Mash.new
   # Single-Service Roles
   # When enabled, default enabled services are disabled
@@ -231,6 +232,7 @@ module Gitlab
       %w(
         registry
+        repmgr
       ).each do |key|
         rkey = key.tr('_', '-')
         results[rkey] = Gitlab[key]

files/gitlab-cookbooks/gitlab/libraries/matchers.rb

@@ -2,4 +2,11 @@ if defined?(ChefSpec)
   def create_templatesymlink(message)
     ChefSpec::Matchers::ResourceMatcher.new(:templatesymlink, :create, message)
   end
+
+  # postgresql_database custom resource matchers
+  ChefSpec.define_matcher :postgresql_database
+
+  def create_postgresql_database(owner)
+    ChefSpec::Matchers::ResourceMatcher.new(:postgresql_database, :create, owner)
+  end
 end

files/gitlab-cookbooks/gitlab/resources/postgresql_database.rb

+resource_name :postgresql_database
+
+property :database, String, name_property: true
+property :owner, String, default: lazy { node['gitlab']['postgresql']['user'] }
+property :helper, default: lazy { PgHelper.new(node) }
+property :database_port, Integer, default: lazy { node['gitlab']['postgresql']['port'] }
+property :database_socket, String, default: lazy { node['gitlab']['postgresql']['unix_socket_directory'] }
+
+action :create do
+  account_helper = AccountHelper.new(node)
+
+  execute "create database #{database}" do
+    command %(/opt/gitlab/embedded/bin/createdb --port #{database_port} -h #{database_socket} -O #{owner} #{database})
+    user account_helper.postgresql_user
+    not_if { !helper.is_running? || helper.database_exists?(database) }
+  end
+end

files/gitlab-cookbooks/repmgr/attributes/default.rb

+default['repmgr']['cluster'] = 'gitlab_cluster'
+default['repmgr']['database'] = 'gitlab_repmgr'
+default['repmgr']['host'] = nil
+default['repmgr']['node_number'] = nil
+default['repmgr']['port'] = 5432
+default['repmgr']['trust_auth_cidr_addresses'] = []
+default['repmgr']['user'] = 'gitlab_repmgr'

files/gitlab-cookbooks/repmgr/attributes/external.rb

+default['gitlab']['postgresql']['custom_pg_hba_entries']['repmgr'] = []

files/gitlab-cookbooks/repmgr/libraries/repmgr_helper.rb

+class RepmgrHelper
+  def initialize(node)
+    @node = node
+  end
+
+  def pg_hba_entries
+    results = []
+    replication_user = @node['repmgr']['user']
+    %W(replication #{@node['repmgr']['database']}).each do |db|
+      results.push(
+        *[
+          {
+            type: 'local',
+            database: db,
+            user: replication_user,
+            method: 'trust'
+          },
+          {
+            type: 'host',
+            database: db,
+            user: replication_user,
+            cidr: '127.0.0.1/32',
+            method: 'trust'
+          }
+        ]
+      )
+
+      @node['repmgr']['trust_auth_cidr_addresses'].each do |addr|
+        results.push(
+          {
+            type: 'host',
+            database: db,
+            user: replication_user,
+            cidr: addr,
+            method: 'trust'
+          }
+        )
+      end
+    end
+    results
+  end
+end

files/gitlab-cookbooks/repmgr/metadata.rb

+name 'repmgr'
+maintainer 'GitLab.com'
+maintainer_email 'support@gitlab.com'
+license 'Apache 2.0'
+description 'Installs/Configures repmgr for GitLab'
+long_description 'Installs/Configures repmgr GitLab'
+version '0.1.0'
+chef_version '>= 12.1' if respond_to?(:chef_version)
+
+issues_url 'https://gitlab.com/gitlab-org/omnibus-gitlab/issues'
+source_url 'https://gitlab.com/gitlab-org/omnibus-gitlab'

files/gitlab-cookbooks/repmgr/recipes/disable.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+runit_service 'repmgr' do
+  action :disable
+end

files/gitlab-cookbooks/repmgr/recipes/enable.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+repmgr_helper = RepmgrHelper.new(node)
+replication_user = node['repmgr']['user']
+repmgr_conf = "#{node['gitlab']['postgresql']['dir']}/repmgr.conf"
+
+node.default['gitlab']['postgresql']['custom_pg_hba_entries']['repmgr'] = repmgr_helper.pg_hba_entries
+
+# node number needs to be unique (to the cluster) positive 32 bit integer.
+# If the user doesn't provide one, generate one ourselves.
+node_number = node['repmgr']['node_number'] ||
+  Digest::MD5.hexdigest(node['fqdn']).unpack('L').first
+template repmgr_conf do
+  source 'repmgr.conf.erb'
+  owner node['gitlab']['postgresql']['username']
+  variables(
+    node['repmgr'].to_hash.merge(
+      node_name: node['repmgr']['node_name'] || node['fqdn'],
+      host: node['repmgr']['host'] || node['fqdn'],
+      node_number: node_number
+    )
+  )
+end
+
+postgresql_user replication_user do
+  options %w(SUPERUSER)
+end
+
+postgresql_database node['repmgr']['database'] do
+  owner replication_user
+  notifies :run, "execute[register repmgr master node]"
+end
+
+execute 'register repmgr master node' do
+  command "/opt/gitlab/embedded/bin/repmgr -f #{repmgr_conf} master register"
+  user node['gitlab']['postgresql']['username']
+  action :nothing
+end

files/gitlab-cookbooks/repmgr/templates/default/repmgr.conf.erb

+cluster=<%= @cluster %>
+node=<%= @node_number %>
+node_name=<%= @node_name %>
+conninfo='host=<%= @host %> port=<%= @port %> user=<%= @user %> dbname=<%= @database %>'
+pg_bindir='/opt/gitlab/embedded/bin'
+service_start_command = '/opt/gitlab/bin/gitlab-ctl start postgresql'
+service_stop_command = '/opt/gitlab/bin/gitlab-ctl stop postgresql'
+service_restart_command = '/opt/gitlab/bin/gitlab-ctl restart postgresql'
+promote_command = '/opt/gitlab/embedded/bin/repmgr standby promote -f /var/opt/gitlab/postgresql/repmgr.conf'
+follow_command = '/opt/gitlab/embedded/bin/repmgr standby follow -f /var/opt/gitlab/postgresql/repmgr.conf'

files/gitlab-ctl-commands-ee/lib/repmgr.rb

+require 'mixlib/shellout'
+require 'timeout'
+
+class RepmgrHelper
+  attr_accessor :command, :subcommand, :args
+
+  def initialize(command, subcommand, args = nil)
+    @command = Kernel.const_get("#{self.class}::#{command.capitalize}")
+    @subcommand = subcommand
+    @args = args
+  end
+
+  def execute
+    @command.send(subcommand, @args)
+  end
+
+  class Base
+    class << self
+      def repmgr_cmd(args, command)
+        cmd("/opt/gitlab/embedded/bin/repmgr #{args[:verbose]} -f /var/opt/gitlab/postgresql/repmgr.conf #{command}", 'gitlab-psql')
+      end
+
+      def cmd(command, user = 'root')
+        results = Mixlib::ShellOut.new(command, user: user, cwd: '/tmp')
+        begin
+          results.run_command
+          results.error!
+        rescue Mixlib::ShellOut::ShellCommandFailed
+          puts "Error running command: #{results.command}"
+          puts "STDOUT: #{results.stdout}" if results.stdout
+          puts "STDERR: #{results.stderr}" if results.stderr
+          raise
+        rescue StandardError => se
+          puts "Unknown Error: #{se}"
+        end
+        results.stdout
+      end
+
+      def repmgr_with_args(command, args)
+        repmgr_cmd(args, "-h #{args[:primary]} -U #{args[:user]} -d #{args[:database]} -D #{args[:directory]} #{command}")
+      end
+
+      def wait_for_postgresql(timeout)
+        # wait for *timeout* seconds for postgresql to respond to queries
+        Timeout.timeout(timeout) do
+          results = nil
+          loop do
+            begin
+              results = cmd("gitlab-psql -l")
+            rescue Mixlib::ShellOut::ShellCommandFailed
+              sleep 1
+              next
+            else
+              break
+            end
+          end
+        end
+      rescue Timeout::TimeoutError
+        raise TimeoutError("Timed out waiting for PostgreSQL to start")
+      end
+    end
+  end
+
+  class Standby < Base
+    class << self
+      def clone(args)
+        repmgr_with_args('standby clone', args)
+      end
+
+      def follow(args)
+        repmgr_with_args('standby follow', args)
+      end
+
+      def promote(args)
+        repmgr_cmd(args, 'standby promote')
+      end
+
+      def register(args)
+        repmgr_cmd(args, 'standby register')
+      end
+
+      def unregister(args, node = nil)
+        return repmgr_cmd(args, "standby unregister --node=#{node}") unless node.nil?
+        repmgr_cmd(args, "standby unregister")
+      end
+
+      def setup(args)
+        if args[:wait]
+          $stdout.puts "Doing this will delete the entire contents of #{args[:directory]}"
+          $stdout.puts "If this is not what you want, hit Ctrl-C now to exit"
+          $stdout.puts "To skip waiting, rerun with the -w option"
+          $stdout.puts "Sleeping for 30 seconds"
+          sleep 30
+        end
+        $stdout.puts "Stopping the database"
+        cmd("gitlab-ctl stop postgresql")
+        $stdout.puts "Removing the data"
+        cmd("rm -rf /var/opt/gitlab/postgresql/data")
+        $stdout.puts "Cloning the data"
+        clone(args)
+        $stdout.puts "Starting the database"
+        cmd("gitlab-ctl start postgresql")
+        # Wait until postgresql is responding to queries before proceeding
+        wait_for_postgresql(30)
+        $stdout.puts "Registering the node with the cluster"
+        register(args)
+      end
+    end
+  end
+
+  class Cluster < Base
+    class << self
+      def show(args)
+        repmgr_cmd(args, 'cluster show')
+      end
+    end
+  end
+
+  class Master < Base
+    class << self
+      def register(args)
+        repmgr_cmd(args, 'master register')
+      end
+    end
+  end
+end

files/gitlab-ctl-commands-ee/repmgr.rb

+require "#{base_path}/embedded/service/omnibus-ctl/lib/repmgr"
+require "#{base_path}/embedded/service/omnibus-ctl/lib/gitlab_ctl"
+
+add_command_under_category('repmgr', 'database', 'Manage repmgr PostgreSQL cluster nodes', 2) do |_cmd_name, _args|
+  repmgr_command = ARGV[3]
+  repmgr_subcommand = ARGV[4]
+  repmgr_primary = ARGV[5]
+  begin
+    node_attributes = GitlabCtl::Util.get_node_attributes(base_path)
+  rescue GitlabCtl::Errors::NodeError => e
+    log e.message
+  end
+  # We only need the arguments if we're performing an action which needs to
+  # know the primary node
+  repmgr_options = repmgr_parse_options
+
+  repmgr_args = begin
+                  {
+                    primary: repmgr_primary,
+                    user: node_attributes['repmgr']['user'],
+                    database:  node_attributes['repmgr']['database'],
+                    directory: node_attributes['gitlab']['postgresql']['data_dir'],
+                    verbose: repmgr_options[:verbose],
+                    wait: repmgr_options[:wait]
+                  }
+                rescue NoMethodError
+                  $stderr.puts "Unable to determine node attributes. Has reconfigure successfully ran?"
+                  exit 1
+                end
+  begin
+    repmgr_obj = RepmgrHelper.new(repmgr_command, repmgr_subcommand, repmgr_args)
+    results = repmgr_obj.execute
+  rescue Mixlib::ShellOut::ShellCommandFailed
+    exit 1
+  rescue NoMethodError
+    if repmgr_command
+      $stderr.puts "The repmgr command #{repmgr_command} does not support #{repmgr_subcommand}"
+    end
+    puts repmgr_help
+    exit 1
+  rescue NameError => ne
+    puts ne
+    $stderr.puts "There is no repmgr command #{repmgr_command}"
+    puts repmgr_help
+    exit 1
+  end
+  log results
+end
+
+def repmgr_help
+  <<-EOF
+  Available repmgr commands:
+  master register -- Register the current node as a master node in the repmgr cluster
+  standby
+    clone MASTER -- Clone the data from node MASTER to set this node up as a standby server
+    register -- Register the node as a standby node in the cluster. Assumes clone has been done
+    setup MASTER -- Performs all steps necessary to setup the current node as a standby for MASTER
+    follow MASTER -- Follow the new master node MASTER
+    unregister --node=X -- Removes the node with id X from the cluster. Without --node removes the current node.
+    promote -- Promote the current node to be the master node
+  cluster show -- Displays the current membership status of the cluster
+  EOF
+end
+
+def repmgr_parse_options
+  options = {
+    wait: true,
+    verbose: ''
+  }
+
+  OptionParser.new do |opts|
+    opts.on('-w', '--no-wait', 'Do not wait before starting the setup process') do
+      options[:wait] = false
+    end
+
+    opts.on('-v', '--verbose', 'Run repmgr with verbose option') do
+      options[:verbose] = '-v'
+    end
+  end.parse!(ARGV)
+
+  options
+end

files/gitlab-ctl-commands/lib/gitlab_ctl.rb

@@ -28,5 +28,7 @@ module GitlabCtl
         @stderr = stderr
       end
     end
+
+    class NodeError < StandardError; end
   end
 end

files/gitlab-ctl-commands/lib/gitlab_ctl/util.rb

 require 'mixlib/shellout'
+require 'chef/mash'
+require 'chef/mixins'
+
+require 'socket'
 module GitlabCtl
   module Util
@@ -24,6 +28,27 @@ module GitlabCtl
         shell_out.run_command
         shell_out
       end
+
+      def fqdn
+        results = run_command('hostname -f')
+        results.stdout.chomp
+      end
+
+      def get_node_attributes(base_path)
+        # reconfigure creates a json file containing all of the attributes of
+        # the node after a chef run, indexed by priority. Merge an return those
+        # as a single level Hash
+        attribute_file = "#{base_path}/embedded/nodes/#{fqdn}.json"
+        begin
+          data = JSON.parse(File.read(attribute_file))
+        rescue JSON::ParserError
+          raise GitlabCtl::Errors::NodeError(
+            "Error reading #{attribute_file}, has reconfigure been run yet?"
+          )
+        end
+
+        Chef::Mixin::DeepMerge.merge(data['default'], data['normal'])
+      end
     end
   end
 end