Merge branch 'bjk/alertmanager' into 'master'

Add Prometheus alertmanager

Closes #2999

See merge request gitlab-org/omnibus-gitlab!2205

.custom_sources.yml

@@ -31,6 +31,9 @@ rb-readline:
 prometheus:
   remote: "git@dev.gitlab.org:omnibus-mirror/prometheus.git"
   alternative: "https://gitlab.com/gitlab-org/build/omnibus-mirror/prometheus.git"
+alertmanager:
+  remote: "git@dev.gitlab.org:omnibus-mirror/alertmanager.git"
+  alternative: "https://gitlab.com/gitlab-org/build/omnibus-mirror/alertmanager.git"
 node-exporter:
   remote: "git@dev.gitlab.org:omnibus-mirror/node_exporter.git"
   alternative: "https://gitlab.com/gitlab-org/build/omnibus-mirror/node_exporter.git"

CHANGELOG.md

@@ -7,6 +7,7 @@ omnibus-gitlab repository.
 - Force kill unicorn process if it is still running after a TERM or INT signal
 - Upgrade Ruby version to 2.3.7
+- Add Prometheus Alertmanager
 - Bump git to 2.16.3
 - Bump curl to 7.59.0 aab088263
 - Bump pcre to 8.42 07c83a5ab

config/projects/gitlab.rb

@@ -101,6 +101,7 @@ dependency 'gitlab-scripts'
 dependency 'gitlab-config-template'
 dependency 'mattermost'
 dependency 'prometheus'
+dependency 'alertmanager'
 dependency 'node-exporter'
 dependency 'redis-exporter'
 dependency 'postgres-exporter'

config/software/alertmanager.rb

+#
+## Copyright:: Copyright (c) 2018 GitLab.com
+## License:: Apache License, Version 2.0
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+#
+
+require "#{Omnibus::Config.project_root}/lib/gitlab/version"
+
+name 'alertmanager'
+version = Gitlab::Version.new('alertmanager', '0.14.0')
+default_version version.print
+
+license 'APACHE-2.0'
+license_file 'LICENSE'
+
+source git: version.remote
+
+relative_path 'src/github.com/prometheus/alertmanager'
+
+build do
+  env = {
+    'GOPATH' => "#{Omnibus::Config.source_dir}/alertmanager",
+  }
+  exporter_source_dir = "#{Omnibus::Config.source_dir}/alertmanager"
+  cwd = "#{exporter_source_dir}/src/github.com/prometheus/alertmanager"
+
+  command 'go build ./cmd/alertmanager', env: env, cwd: cwd
+  copy 'alertmanager', "#{install_dir}/embedded/bin/"
+end

files/gitlab-config-template/gitlab.rb.template

@@ -1410,6 +1410,24 @@ external_url 'GENERATED_EXTERNAL_URL'
 ##! Advanced settings. Should be changed only if absolutely needed.
 # prometheus['listen_address'] = 'localhost:9090'
+################################################################################
+## Prometheus Alertmanager
+##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/alertmanager.html
+################################################################################
+
+# alertmanager['enable'] = true
+# alertmanager['home'] = '/var/opt/gitlab/alertmanager'
+# alertmanager['log_directory'] = '/var/log/gitlab/alertmanager'
+# alertmanager['admin_email'] = 'admin@example.com'
+# alertmanager['flags'] = {
+#   'web.listen-address' => "#{node['gitlab']['alertmanager']['listen_address']}"
+#   'storage.path' => "#{node['gitlab']['alertmanager']['home']}/data"
+#   'config.file' => "#{node['gitlab']['alertmanager']['home']}/alertmanager.yml"
+# }
+
+##! Advanced settings. Should be changed only if absolutely needed.
+# alertmanager['listen_address'] = 'localhost:9093'
+
 ################################################################################
 ## Prometheus Node Exporter
 ##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/node_exporter.html

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -777,6 +777,20 @@ default['gitlab']['prometheus']['target_heap_size'] = (
   26_214_400 + (node['memory']['total'].to_i * 1024 * 0.02)
 ).to_i
+####
+# Prometheus Alertmanager
+####
+
+default['gitlab']['alertmanager']['enable'] = false
+default['gitlab']['alertmanager']['home'] = '/var/opt/gitlab/alertmanager'
+default['gitlab']['alertmanager']['log_directory'] = '/var/log/gitlab/alertmanager'
+default['gitlab']['alertmanager']['listen_address'] = 'localhost:9093'
+default['gitlab']['alertmanager']['admin_email'] = nil
+default['gitlab']['alertmanager']['inhibit_rules'] = []
+default['gitlab']['alertmanager']['receivers'] = []
+default['gitlab']['alertmanager']['routes'] = []
+default['gitlab']['alertmanager']['templates'] = []
+
 ####
 # Prometheus Node Exporter
 ####

files/gitlab-cookbooks/gitlab/libraries/logging.rb

@@ -36,6 +36,7 @@ module Logging
       end
       %w(
+        alertmanager
         geo-logcursor
         geo-postgresql
         gitaly

files/gitlab-cookbooks/gitlab/libraries/prometheus.rb

@@ -30,6 +30,7 @@ module Prometheus
     def parse_variables
       parse_exporter_enabled
       parse_monitoring_enabled
+      parse_alertmanager_config
       parse_scrape_configs
       parse_flags
     end
@@ -47,6 +48,7 @@ module Prometheus
     def parse_flags
       parse_prometheus_flags
+      parse_alertmanager_flags
       parse_node_exporter_flags
       parse_postgres_exporter_flags
       parse_redis_exporter_flags
@@ -73,6 +75,24 @@ module Prometheus
       Gitlab['prometheus']['flags'] = default_config['flags']
     end
+    def parse_alertmanager_flags
+      default_config = Gitlab['node']['gitlab']['alertmanager'].to_hash
+      user_config = Gitlab['alertmanager']
+
+      home_directory = user_config['home'] || default_config['home']
+      listen_address = user_config['listen_address'] || default_config['listen_address']
+
+      default_config['flags'] = {
+        'web.listen-address' => listen_address,
+        'storage.path' => File.join(home_directory, 'data'),
+        'config.file' => File.join(home_directory, 'alertmanager.yml')
+      }
+
+      default_config['flags'].merge!(user_config['flags']) if user_config.key?('flags')
+
+      Gitlab['alertmanager']['flags'] = default_config['flags']
+    end
+
     def parse_node_exporter_flags
       default_config = Gitlab['node']['gitlab']['node-exporter'].to_hash
       user_config = Gitlab['node_exporter']
@@ -120,6 +140,39 @@ module Prometheus
       Gitlab['postgres_exporter']['flags'] = default_config['flags']
     end
+    def parse_alertmanager_config
+      user_config = Gitlab['alertmanager']
+      rails_config = Gitlab['gitlab_rails']
+
+      global = {}
+
+      if rails_config['smtp_enable']
+        global['smtp_from'] = rails_config['gitlab_email_from'] || 'unconfigured'
+        global['smtp_smarthost'] = "#{rails_config['smtp_address'] || 'unconfigured'}:#{rails_config['smtp_port'] || '25'}"
+        if %w(login plain).include?(rails_config['smtp_authentication'])
+          global['smtp_auth_username'] = rails_config['smtp_user_name']
+          global['smtp_auth_password'] = rails_config['smtp_password']
+        end
+      end
+
+      default_email_receiver = {
+        'name' => 'default-receiver',
+      }
+      default_email_receiver['email_configs'] = ['to' => user_config['admin_email']] unless user_config['admin_email'].nil?
+
+      default_inhibit_rules = [] << Gitlab['alertmanager']['inhibit_rules']
+      default_receivers = [] << default_email_receiver << Gitlab['alertmanager']['receivers']
+      default_routes = [] << Gitlab['alertmanager']['routes']
+      default_templates = [] << Gitlab['alertmanager']['templates']
+
+      Gitlab['alertmanager']['global'] = global
+      Gitlab['alertmanager']['inhibit_rules'] = default_inhibit_rules.compact.flatten
+      Gitlab['alertmanager']['receivers'] = default_receivers.compact.flatten
+      Gitlab['alertmanager']['routes'] = default_routes.compact.flatten
+      Gitlab['alertmanager']['templates'] = default_templates.compact.flatten
+      Gitlab['alertmanager']['default_receiver'] = user_config['default_receiver'] || 'default-receiver'
+    end
+
     def parse_scrape_configs
       # Don't parse if prometheus is explicitly disabled
       return unless Services.enabled?('prometheus')

files/gitlab-cookbooks/gitlab/recipes/alertmanager.rb

+#
+# Copyright:: Copyright (c) 2018 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+account_helper = AccountHelper.new(node)
+prometheus_user = account_helper.prometheus_user
+alertmanager_log_dir = node['gitlab']['alertmanager']['log_directory']
+alertmanager_dir = node['gitlab']['alertmanager']['home']
+
+# alertmanager runs under the prometheus user account. If prometheus is
+# disabled, it's up to this recipe to create the account
+include_recipe 'gitlab::prometheus_user'
+
+directory alertmanager_dir do
+  owner prometheus_user
+  mode '0750'
+  recursive true
+end
+
+directory alertmanager_log_dir do
+  owner prometheus_user
+  mode '0700'
+  recursive true
+end
+
+configuration = {
+  'global' => node['gitlab']['alertmanager']['global'],
+  'templates' => node['gitlab']['alertmanager']['templates'],
+  'route' => {
+    'receiver' => node['gitlab']['alertmanager']['default_receiver'],
+    'routes' => node['gitlab']['alertmanager']['routes'],
+  },
+  'receivers' => node['gitlab']['alertmanager']['receivers'],
+  'inhibit_rules' => node['gitlab']['alertmanager']['inhibit_rules'],
+}
+
+file 'Alertmanager config' do
+  path File.join(alertmanager_dir, 'alertmanager.yml')
+  content Prometheus.hash_to_yaml(configuration)
+  owner prometheus_user
+  mode '0644'
+  notifies :restart, 'service[alertmanager]'
+end
+
+runtime_flags = PrometheusHelper.new(node).kingpin_flags('alertmanager')
+runit_service 'alertmanager' do
+  options({
+    log_directory: alertmanager_log_dir,
+    flags: runtime_flags
+  }.merge(params))
+  log_options node['gitlab']['logging'].to_hash.merge(
+    node['gitlab']['alertmanager'].to_hash
+  )
+end
+
+if node['gitlab']['bootstrap']['enable']
+  execute "/opt/gitlab/bin/gitlab-ctl start alertmanager" do
+    retries 20
+  end
+end

files/gitlab-cookbooks/gitlab/recipes/alertmanager_disable.rb

+#
+# Copyright:: Copyright (c) 2018 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+runit_service 'alertmanager' do
+  action :disable
+end

files/gitlab-cookbooks/gitlab/templates/default/sv-alertmanager-log-config.erb

+<%= "s#@svlogd_size" if @svlogd_size %>
+<%= "n#@svlogd_num" if @svlogd_num %>
+<%= "t#@svlogd_timeout" if @svlogd_timeout %>
+<%= "!#@svlogd_filter" if @svlogd_filter %>
+<%= "u#@svlogd_udp" if @svlogd_udp %>
+<%= "p#@svlogd_prefix" if @svlogd_prefix %>

files/gitlab-cookbooks/gitlab/templates/default/sv-alertmanager-log-run.erb

+#!/bin/sh
+exec svlogd -tt <%= @options[:log_directory] %>

files/gitlab-cookbooks/gitlab/templates/default/sv-alertmanager-run.erb

+#!/bin/sh
+exec 2>&1
+<%= render("mount_point_check.erb") %>
+umask 077
+exec chpst -P -U <%= node['gitlab']['prometheus']['username'] %> -u <%= node['gitlab']['prometheus']['username'] %> \
+  /opt/gitlab/embedded/bin/alertmanager <%= @options[:flags] %>

files/gitlab-cookbooks/package/libraries/config/gitlab.rb

@@ -89,6 +89,7 @@ module Gitlab
     attribute('logrotate')
     attribute('high_availability')
     attribute('web_server')
+    attribute('alertmanager')
     attribute('node_exporter')
     attribute('redis_exporter')
     attribute('postgres_exporter')

files/gitlab-cookbooks/package/libraries/config/services.rb

@@ -32,6 +32,7 @@ module Services
     service 'postgresql',         groups: [DEFAULT_GROUP, 'postgres', 'postgres_role']
     service 'nginx',              groups: [DEFAULT_GROUP]
     service 'prometheus',         groups: [DEFAULT_GROUP, 'prometheus']
+    service 'alertmanager',       groups: [DEFAULT_GROUP, 'prometheus']
     service 'postgres_exporter',  groups: [DEFAULT_GROUP, 'prometheus', 'postgres', 'postgres_role']
     service 'mailroom'
     service 'gitlab_pages'

spec/chef/gitlab-ee/recipes/geo-secondary_spec.rb

@@ -88,6 +88,7 @@ describe 'gitlab-ee::geo-secondary' do
       stub_gitlab_rb(geo_secondary_role: { enable: true })
       %w(
+        alertmanager
         gitlab-monitor
         gitlab-workhorse
         logrotate

spec/chef/gitlab-ee/recipes/geo_database_migrations_spec.rb

@@ -20,6 +20,7 @@ describe 'gitlab-ee::geo-database-migrations' do
       stub_gitlab_rb(geo_secondary_role: { enable: true })
       %w(
+        alertmanager
         gitlab-monitor
         gitlab-workhorse
         logrotate

spec/chef/recipes/alertmanager_spec.rb

+require 'chef_helper'
+
+alertmanager_yml_output = <<-ALERTMANAGERYML
+  ---
+  global:
+    smtp_from: gitlab-omnibus
+    smtp_smarthost: testhost:25
+  templates: []
+  route:
+    receiver: default-receiver
+    routes: []
+  receivers:
+  - name: default-receiver
+    email_configs:
+    - to: admin@example.com
+  inhibit_rules: []
+ALERTMANAGERYML
+
+describe 'gitlab::alertmanager' do
+  let(:chef_run) { ChefSpec::SoloRunner.new(step_into: %w(account)).converge('gitlab::default') }
+
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
+
+  context 'when alertmanager is enabled' do
+    let(:config_template) { chef_run.template('/var/log/gitlab/alertmanager/config') }
+
+    before do
+      stub_gitlab_rb(
+        prometheus: { enable: true },
+        alertmanager: {
+          enable: true,
+          admin_email: 'admin@example.com',
+        },
+        gitlab_rails: {
+          gitlab_email_from: 'gitlab-omnibus',
+          smtp_enable: true,
+          smtp_address: 'testhost',
+          smtp_port: 25,
+        }
+      )
+    end
+
+    it_behaves_like 'enabled runit service', 'alertmanager', 'root', 'root'
+
+    it 'populates the files with expected configuration' do
+      expect(config_template).to notify('ruby_block[reload alertmanager svlogd configuration]')
+
+      expect(chef_run).to render_file('/opt/gitlab/sv/alertmanager/run')
+        .with_content { |content|
+          expect(content).to match(/exec chpst -P/)
+          expect(content).to match(/\/opt\/gitlab\/embedded\/bin\/alertmanager/)
+          expect(content).to match(/alertmanager.yml/)
+        }
+
+      expect(chef_run).to render_file('/var/opt/gitlab/alertmanager/alertmanager.yml')
+        .with_content(alertmanager_yml_output.gsub(/^ {2}/, ''))
+
+      expect(chef_run).to render_file('/opt/gitlab/sv/alertmanager/log/run')
+        .with_content(/exec svlogd -tt \/var\/log\/gitlab\/alertmanager/)
+    end
+
+    it 'creates default set of directories' do
+      expect(chef_run).to create_directory('/var/log/gitlab/alertmanager').with(
+        owner: 'gitlab-prometheus',
+        group: nil,
+        mode: '0700'
+      )
+      expect(chef_run).to create_directory('/var/opt/gitlab/alertmanager').with(
+        owner: 'gitlab-prometheus',
+        group: nil,
+        mode: '0750'
+      )
+    end
+
+    it 'should create a gitlab-prometheus user account' do
+      expect(chef_run).to create_user('gitlab-prometheus')
+    end
+
+    it 'sets a default listen address' do
+      expect(chef_run).to render_file('/opt/gitlab/sv/alertmanager/run')
+        .with_content(/web.listen-address=localhost:9093/)
+    end
+  end
+
+  context 'with user provided settings' do
+    before do
+      stub_gitlab_rb(
+        prometheus: { enable: true },
+        alertmanager: {
+          listen_address: ':9093',
+          enable: true,
+        },
+        gitlab_rails: {
+          smtp_enable: true,
+          smtp_address: 'other-testhost',
+          smtp_port: 465,
+        }
+      )
+    end
+
+    it 'populates the files with expected configuration' do
+      expect(chef_run).to render_file('/opt/gitlab/sv/alertmanager/run')
+        .with_content(/web.listen-address=:9093/)
+    end
+
+    it 'keeps the defaults that the user did not override' do
+      expect(chef_run).to render_file('/var/opt/gitlab/alertmanager/alertmanager.yml')
+        .with_content(/receiver: default-receiver/)
+    end
+
+    it 'renders alertmanager.yml with the non-default value' do
+      expect(chef_run).to render_file('/var/opt/gitlab/alertmanager/alertmanager.yml')
+        .with_content(/smtp_smarthost: other-testhost:465/)
+    end
+  end
+end

spec/chef/recipes/config_spec.rb

@@ -59,6 +59,7 @@ describe 'gitlab::config' do
       expect(node['gitlab']['postgresql']['enable']).to eq true
       expect(node['gitlab']['redis']['enable']).to eq true
       expect(node['gitlab']['prometheus']['enable']).to eq true
+      expect(node['gitlab']['alertmanager']['enable']).to eq true
       expect(node['gitlab']['node-exporter']['enable']).to eq true
       expect(node['gitlab']['redis-exporter']['enable']).to eq true
       expect(node['gitlab']['logrotate']['enable']).to eq true

spec/chef/recipes/gitlab-rails_spec.rb

@@ -757,6 +757,7 @@ describe 'gitlab::gitlab-rails' do
     before do
       %w(
+        alertmanager
         gitlab-monitor
         gitlab-workhorse
         logrotate