Adding consul to EE build

config/projects/gitlab.rb

@@ -76,6 +76,7 @@ dependency 'redis'
 dependency 'nginx'
 dependency 'mixlib-log'
 dependency 'chef-zero'
+dependency 'consul' if ee
 dependency 'ohai'
 dependency 'chef-gem'
 dependency 'remote-syslog'

config/software/consul.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name 'consul'
+default_version 'v0.9.0'
+
+license 'MPL-2.0'
+license_file 'LICENSE'
+
+version '0.9.0' do
+  source sha256: '4e3db525b58ba9ed8d3f0a09047d4935180748f44be2a48342414bfcff3c69a4'
+end
+
+source git: 'https://github.com/hashicorp/consul.git'
+
+relative_path 'src/github.com/hashicorp/consul'
+
+build do
+  env = {}
+  env['GOPATH'] = "#{Omnibus::Config.source_dir}/consul"
+  env['PATH'] = "#{ENV['PATH']}:#{env['GOPATH']}/bin"
+  command 'make dev', env: env
+  copy 'bin/consul', "#{install_dir}/embedded/bin/"
+end

config/software/gitlab-ctl-ee.rb

@@ -28,11 +28,5 @@ source path: File.expand_path(
 )
 build do
-  if ee
-    ['', 'lib/'].each do |dir|
-      copy File.expand_path(
-        "files/gitlab-ctl-commands-ee/#{dir}*.rb", Omnibus::Config.project_root
-      ), "#{install_dir}/embedded/service/omnibus-ctl/#{dir}"
-    end
-  end
+  sync './', "#{install_dir}/embedded/service/omnibus-ctl-ee/" if ee
 end

config/software/gitlab-ctl.rb

@@ -63,7 +63,7 @@ done
 # As documented at http://smarden.org/runit/sv.8.html
 export SVWAIT=30
-#{install_dir}/embedded/bin/omnibus-ctl #{File.basename(install_dir)} #{install_dir}/embedded/service/omnibus-ctl $@
+#{install_dir}/embedded/bin/omnibus-ctl #{File.basename(install_dir)} '#{install_dir}/embedded/service/omnibus-ctl*' $@
        EOH
     end
   end

doc/package-information/defaults.md

@@ -37,6 +37,7 @@ by default:
 | Mattermost       | No            | Port             | X           | 8065                                   |
 | Mattermost       | No            | Port             | X           | 80 or 443                              |
 | PgBouncer        | No            | Port             | X           | 6432                                   |
+| Consul           | No            | Port             | X           | 8300, 8500                             |
 Legend:
 * `Component` - Name of the component.

doc/settings/configuration.md

@@ -267,6 +267,9 @@ mattermost
 # GitLab Registry user (only when using GitLab Registry)
 registry
+
+# GitLab Consul user (only when using GitLab Consul)
+gitlab-consul
 ```
 By default, omnibus-gitlab package expects that following groups exist:
@@ -292,6 +295,9 @@ mattermost
 # GitLab Registry group (only when using GitLab Registry)
 registry
+
+# GitLab Consul group (only when using GitLab Consul)
+gitlab-consul
 ```
 You can also use different user/group names but then you must specify user/group details in `/etc/gitlab/gitlab.rb`, eg.

docker/assets/setup

@@ -34,6 +34,7 @@ groupadd -g 996 gitlab-psql
 groupadd -g 994 mattermost
 groupadd -g 993 registry
 groupadd -g 992 gitlab-prometheus
+groupadd -g 991 gitlab-consul
 # Create accounts
 useradd -m -u 998 -g git -m -s /bin/sh -d /var/opt/gitlab git
@@ -43,3 +44,4 @@ useradd -m -u 996 -g gitlab-psql -m -s /bin/sh -d /var/opt/gitlab/postgresql git
 useradd -m -u 994 -g mattermost -m -s /bin/sh -d /var/opt/gitlab/mattermost mattermost
 useradd -m -u 993 -g registry -m -s /bin/sh -d /var/opt/gitlab/registry registry
 useradd -m -u 992 -g gitlab-prometheus -m -s /bin/sh -d /var/opt/gitlab/prometheus gitlab-prometheus
+useradd -m -u 991 -g gitlab-consul -m -s /bin/sh -d /var/opt/gitlab/consul gitlab-consul

files/gitlab-config-template/gitlab.rb.template

@@ -1557,6 +1557,12 @@ external_url 'GENERATED_EXTERNAL_URL'
 # pgbouncer['auth_type'] = 'md5'
 # pgbouncer['auth_hba_file'] = nil
 # pgbouncer['auth_query'] = 'SELECT username, password FROM public.pg_shadow_lookup($1)'
+# pgbouncer['users'] = [
+#   {
+#     name: USERNAME,
+#     password: MD5_PASSWORD_HASH
+#   }
+# ]
 # postgresql['pgbouncer_user'] = nil
 # postgresql['pgbouncer_user_password'] = nil
 #
@@ -1571,3 +1577,43 @@ external_url 'GENERATED_EXTERNAL_URL'
 # repmgr['port'] = 5432
 # repmgr['trust_auth_cidr_addresses'] = []
 # repmgr['user'] = 'gitlab_repmgr'
+
+################################################################################
+# Consul (EEP only)
+################################################################################
+# consul['enable'] = false
+# consul['dir'] = '/var/opt/gitlab/consul'
+# consul['user'] = 'gitlab-consul'
+# consul['config_file'] = '/var/opt/gitlab/consul/config.json'
+# consul['config_dir'] = '/var/opt/gitlab/consul/config.d'
+# consul['data_dir'] = '/var/opt/gitlab/consul/data'
+# consul['log_directory'] = '/var/log/gitlab/consul'
+# consul['script_directory'] = '/var/opt/gitlab/consul/scripts'
+# consul['configuration'] = {
+#   'client_addr' => nil,
+#   'datacenter' => 'gitlab_consul',
+#   'enable_script_checks' => true,
+#   'server' => false
+# }
+# consul['services'] = []
+# consul['service_config'] = {
+#   'postgresql' => {
+#     'service' => {
+#       'name' => "postgresql",
+#       'address' => '',
+#       'port' => 5432,
+#       'checks' => [
+#         {
+#           'script' => "/var/opt/gitlab/consul/scripts/check_postgresql",
+#           'interval' => "10s"
+#         }
+#       ]
+#     }
+#   }
+# }
+# consul['watchers'] = {
+#   'postgresql' => {
+#     enable: false,
+#     handler: 'failover_pgbouncer'
+#   }
+# }

files/gitlab-cookbooks/consul/attributes/default.rb

+default['consul']['enable'] = false
+default['consul']['dir'] = '/var/opt/gitlab/consul'
+default['consul']['user'] = 'gitlab-consul'
+default['consul']['config_file'] = '/var/opt/gitlab/consul/config.json'
+default['consul']['config_dir'] = '/var/opt/gitlab/consul/config.d'
+default['consul']['data_dir'] = '/var/opt/gitlab/consul/data'
+default['consul']['log_directory'] = '/var/log/gitlab/consul'
+default['consul']['script_directory'] = '/var/opt/gitlab/consul/scripts'
+default['consul']['configuration'] = {}

files/gitlab-cookbooks/consul/attributes/services.rb

+default['consul']['services'] = []
+default['consul']['service_config'] = {
+  'postgresql' => {
+    'service' => {
+      'name' => "postgresql",
+      'address' => '',
+      'port' => 5432,
+      'checks' => [
+        {
+          'script' => '/opt/gitlab/bin/gitlab-ctl repmgr-check-master',
+          'interval' => "10s"
+        }
+      ]
+    }
+  }
+}

files/gitlab-cookbooks/consul/attributes/watchers.rb

+default['consul']['watchers'] = []
+default['consul']['watcher_config'] = {
+  'postgresql' => {
+    'handler' => 'failover_pgbouncer'
+  }
+}

files/gitlab-cookbooks/consul/libraries/consul_helper.rb

+class ConsulHelper
+  attr_reader :node, :default_configuration
+
+  def initialize(node)
+    @node = node
+    @default_configuration = {
+      'client_addr' => nil,
+      'datacenter' => 'gitlab_consul',
+      'disable_update_check' => true,
+      'enable_script_checks' => true,
+      'node_name' => node['fqdn'],
+      'rejoin_after_leave' => true,
+      'server' => false
+    }
+  end
+
+  def watcher_config(watcher)
+    {
+      watches: [
+        {
+          type: 'service',
+          service: watcher,
+          handler: "#{node['consul']['script_directory']}/#{node['consul']['watcher_config'][watcher]['handler']}"
+        }
+      ]
+    }
+  end
+
+  def configuration
+    Chef::Mixin::DeepMerge.merge(
+      default_configuration,
+      node['consul']['configuration']
+    ).select { |k, v| !v.nil? }.to_json
+  end
+end

files/gitlab-cookbooks/consul/recipes/disable.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe 'consul::disable_daemon'

files/gitlab-cookbooks/consul/recipes/disable_daemon.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+runit_service 'consul' do
+  action :disable
+end

files/gitlab-cookbooks/consul/recipes/enable.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+account_helper = AccountHelper.new(node)
+consul_helper = ConsulHelper.new(node)
+
+account "Consul user and group" do
+  username account_helper.consul_user
+  uid node['consul']['uid']
+  ugid account_helper.consul_user
+  groupname account_helper.consul_user
+  gid node['consul']['gid']
+  home node['consul']['dir']
+  manage node['gitlab']['manage-accounts']['enable']
+end
+
+directory node['consul']['dir'] do
+  owner account_helper.consul_user
+end
+
+%w(
+  config_dir
+  data_dir
+  log_directory
+  script_directory
+).each do |dir|
+  directory node['consul'][dir] do
+    owner account_helper.consul_user
+  end
+end
+
+# By default consul only listens on the loopback interface.
+# If we're running in server mode then this is not useful
+if node['consul']['configuration']['server']
+  node.default['consul']['configuration']['client_addr'] = node['ipaddress'] unless node['consul']['configuration'].attribute?('client_addr')
+end
+
+file "#{node['consul']['dir']}/config.json" do
+  content consul_helper.configuration
+  owner account_helper.consul_user
+  notifies :restart, "service[consul]"
+end
+
+node['consul']['services'].each do |service|
+  include_recipe "consul::service_#{service}"
+end
+
+include_recipe 'consul::watchers'
+
+include_recipe 'consul::enable_daemon'

files/gitlab-cookbooks/consul/recipes/enable_daemon.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+account_helper = AccountHelper.new(node)
+
+runit_service 'consul' do
+  options({
+            config_dir: node['consul']['config_dir'],
+            config_file: node['consul']['config_file'],
+            data_dir: node['consul']['data_dir'],
+            dir: node['consul']['dir'],
+            log_directory: node['consul']['log_directory'],
+            user: node['consul']['user']
+          })
+  supervisor_owner account_helper.consul_user
+  supervisor_group account_helper.consul_user
+  owner account_helper.consul_user
+  group account_helper.consul_user
+end

files/gitlab-cookbooks/consul/recipes/service_postgresql.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+account_helper = AccountHelper.new(node)
+
+file "#{node['consul']['config_dir']}/postgresql_service.json" do
+  content node['consul']['service_config']['postgresql'].to_json
+  owner account_helper.consul_user
+end
+
+include_recipe 'repmgr::consul_user_permissions'

files/gitlab-cookbooks/consul/recipes/watchers.rb

+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+account_helper = AccountHelper.new(node)
+consul_helper = ConsulHelper.new(node)
+
+node['consul']['watchers'].each do |watcher|
+  config = consul_helper.watcher_config(watcher)
+
+  file "#{node['consul']['config_dir']}/watcher_#{watcher}.json" do
+    content config.to_json
+    owner account_helper.postgresql_user
+  end
+
+  config[:watches].each do |watch|
+    template watch[:handler] do
+      source "watcher_scripts/#{node['consul']['watcher_config'][watch[:service]][:handler]}.erb"
+      variables node['consul'].to_hash
+      mode 0555
+    end
+  end
+end
+
+# Watcher specific settings
+if node['consul']['watchers'].include?('postgresql')
+  node.default['gitlab']['pgbouncer']['databases_ini'] = '/var/opt/gitlab/consul/databases.ini'
+  node.default['gitlab']['pgbouncer']['databases_ini_user'] = 'gitlab-consul'
+end

files/gitlab-cookbooks/consul/templates/default/sv-consul-log-config.erb

+<%= "s#@svlogd_size" if @svlogd_size %>
+<%= "n#@svlogd_num" if @svlogd_num %>
+<%= "t#@svlogd_timeout" if @svlogd_timeout %>
+<%= "!#@svlogd_filter" if @svlogd_filter %>
+<%= "u#@svlogd_udp" if @svlogd_udp %>
+<%= "p#@svlogd_prefix" if @svlogd_prefix %>

files/gitlab-cookbooks/consul/templates/default/sv-consul-log-run.erb

+#!/bin/sh
+exec svlogd -tt <%= @options[:log_directory] %>