Merge branch 'mattkasa-add-gitlab-kas' into 'master'

Add gitlab-kas

Closes gitlab#223060

See merge request gitlab-org/omnibus-gitlab!4579

.custom_sources.yml

@@ -22,6 +22,9 @@ gitlab-pages:
 gitlab-exporter:
   remote: "git@dev.gitlab.org:gitlab-org/gitlab-exporter.git"
   alternative: "https://gitlab.com/gitlab-org/gitlab-exporter.git"
+gitlab-kas:
+  remote: "git@dev.gitlab.org:gitlab/gitlab-agent.git"
+  alternative: "https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent.git"
 docker-distribution-pruner:
   remote: "git@dev.gitlab.org:gitlab/docker-distribution-pruner.git"
   alternative: "https://gitlab.com/gitlab-org/docker-distribution-pruner.git"

GITLAB_KAS_VERSION

+0.0.6

changelogs/unreleased/mkasa-add-gitlab-kas.yml

+---
+title: Add gitlab-kas to omnibus
+merge_request: 4579
+author:
+type: added

config/projects/gitlab.rb

@@ -117,6 +117,7 @@ dependency 'gitlab-shell'
 dependency 'gitlab-ctl'
 dependency 'gitlab-psql'
 dependency 'gitlab-redis-cli'
+dependency 'gitlab-kas'
 dependency 'gitlab-healthcheck'
 dependency 'gitlab-cookbooks'
 dependency 'chef-acme'

config/software/gitlab-kas.rb

+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require "#{Omnibus::Config.project_root}/lib/gitlab/version"
+version = Gitlab::Version.new('gitlab-kas')
+
+name 'gitlab-kas'
+default_version version.print
+
+license 'MIT'
+license_file 'LICENSE'
+
+skip_transitive_dependency_licensing true
+
+source git: version.remote
+
+build do
+  env = { 'TARGET_DIRECTORY' => "#{Omnibus::Config.source_dir}/gitlab-kas/build" }
+
+  make 'kas', env: env
+  move 'build/kas', "#{install_dir}/embedded/bin/gitlab-kas"
+
+  command "license_finder report --decisions-file=#{Omnibus::Config.project_root}/support/dependency_decisions.yml --format=csv --save=license.csv"
+  copy "license.csv", "#{install_dir}/licenses/gitlab-kas.csv"
+end

doc/build/team_member_docs.md

@@ -43,6 +43,7 @@ details:
 | GITLAB_SHELL_VERSION                 | GITLAB_SHELL_VERSION                 | Git reference of the [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell) repository.|
 | GITLAB_WORKHORSE_VERSION             | GITLAB_WORKHORSE_VERSION             | Git reference of the [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) repository.|
 | GITLAB_ELASTICSEARCH_INDEXER_VERSION | GITLAB_ELASTICSEARCH_INDEXER_VERSION | Git reference of the [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer) repository. Used only in EE builds.|
+| GITLAB_KAS_VERSION                   | GITLAB_KAS_VERSION                   | Git reference of the [GitLab Kubernetes Agent Server](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent) repository.|
 If you are running `package-and-qa` job from a GitLab MR, `GITLAB_VERSION`
 environment variable will be set to the commit SHA corresponding to the pipeline

doc/package-information/defaults.md

@@ -51,6 +51,7 @@ by default:
 | <a name="pgbouncer"></a>           PgBouncer           | No            | Port             | X           | 6432                                   |
 | <a name="consul"></a>              Consul              | No            | Port             | X           | 8300, 8500                             |
 | <a name="patroni"></a>             Patroni             | No            | Port             | X           | 8008                                   |
+| <a name="gitlab-kas"></a>          GitLab KAS          | No            | Port             | X           | 8150                                   |
 Legend:

files/gitlab-config-template/gitlab.rb.template

@@ -1535,6 +1535,38 @@ gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = nil
 # gitlab_ci['gitlab_ci_add_pusher'] = true
 # gitlab_ci['builds_directory'] = '/var/opt/gitlab/gitlab-ci/builds'
+################################################################################
+## GitLab Kubernetes Agent Server
+##! Docs: https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/blob/master/README.md
+################################################################################
+
+##! Enable GitLab KAS
+# gitlab_kas['enable'] = true
+
+##! Agent configuration for GitLab KAS
+# gitlab_kas['agent_configuration_poll_period'] = 20
+# gitlab_kas['agent_gitops_poll_period'] = 20
+# gitlab_kas['agent_gitops_project_info_cache_ttl'] = 300
+# gitlab_kas['agent_gitops_project_info_cache_error_ttl'] = 60
+# gitlab_kas['agent_info_cache_ttl'] = 300
+# gitlab_kas['agent_info_cache_error_ttl'] = 60
+
+##! Shared secret used for authentication between KAS and GitLab
+# gitlab_kas['api_secret_key'] = nil # Will be generated if not set. Base64 encoded and exactly 32 bytes long.
+
+##! Listen configuration for GitLab KAS
+# gitlab_kas['listen_address'] = 'localhost:8150'
+# gitlab_kas['listen_network'] = 'tcp'
+# gitlab_kas['listen_websocket'] = true
+
+##! Metrics configuration for GitLab KAS
+# gitlab_kas['metrics_usage_reporting_period'] = 60
+
+##! Directories for GitLab KAS
+# gitlab_kas['dir'] = '/var/opt/gitlab/gitlab-kas'
+# gitlab_kas['log_directory'] = '/var/log/gitlab/gitlab-kas'
+# gitlab_kas['env_directory'] = '/opt/gitlab/etc/gitlab-kas/env'
+
 ################################################################################
 ## GitLab Mattermost
 ##! Docs: https://docs.gitlab.com/omnibus/gitlab-mattermost

files/gitlab-cookbooks/gitlab-kas/attributes/default.rb

+####
+# GitLab Kubernetes Agent Server
+####
+default['gitlab-kas']['enable'] = false
+default['gitlab-kas']['agent_configuration_poll_period'] = 20
+default['gitlab-kas']['agent_gitops_poll_period'] = 20
+default['gitlab-kas']['agent_gitops_project_info_cache_ttl'] = 300
+default['gitlab-kas']['agent_gitops_project_info_cache_error_ttl'] = 60
+default['gitlab-kas']['agent_info_cache_ttl'] = 300
+default['gitlab-kas']['agent_info_cache_error_ttl'] = 60
+default['gitlab-kas']['gitlab_address'] = ''
+default['gitlab-kas']['api_secret_key'] = nil
+default['gitlab-kas']['listen_address'] = 'localhost:8150'
+default['gitlab-kas']['listen_network'] = 'tcp'
+default['gitlab-kas']['listen_websocket'] = true
+default['gitlab-kas']['metrics_usage_reporting_period'] = 60
+default['gitlab-kas']['dir'] = '/var/opt/gitlab/gitlab-kas'
+default['gitlab-kas']['log_directory'] = '/var/log/gitlab/gitlab-kas'
+default['gitlab-kas']['env_directory'] = '/opt/gitlab/etc/gitlab-kas/env'

files/gitlab-cookbooks/gitlab-kas/libraries/gitlab_kas.rb

+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative '../../package/libraries/helpers/secrets_helper'
+
+module GitlabKas
+  class << self
+    def parse_variables
+      parse_address
+    end
+
+    def parse_address
+      Gitlab['gitlab_kas']['gitlab_address'] ||= Gitlab['external_url']
+    end
+
+    def parse_secrets
+      # KAS and GitLab expects exactly 32 bytes, encoded with base64
+      Gitlab['gitlab_kas']['api_secret_key'] ||= Base64.strict_encode64(SecretsHelper.generate_hex(16))
+
+      api_secret_key = Base64.strict_decode64(Gitlab['gitlab_kas']['api_secret_key'])
+      raise "gitlab_kas['api_secret_key'] should be exactly 32 bytes" if api_secret_key.length != 32
+    end
+  end
+end

files/gitlab-cookbooks/gitlab-kas/metadata.rb

+name 'gitlab-kas'
+maintainer 'GitLab.com'
+maintainer_email 'support@gitlab.com'
+license 'Apache 2.0'
+description 'Installs/Configures GitLab KAS'
+long_description 'Installs/Configures the GitLab Kubernetes Agent Server'
+version '0.1.0'
+chef_version '>= 12.1' if respond_to?(:chef_version)
+
+issues_url 'https://gitlab.com/gitlab-org/omnibus-gitlab/issues'
+source_url 'https://gitlab.com/gitlab-org/omnibus-gitlab'
+
+depends 'package'
+depends 'gitlab'

files/gitlab-cookbooks/gitlab-kas/recipes/disable.rb

+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+runit_service 'gitlab-kas' do
+  action :disable
+end

files/gitlab-cookbooks/gitlab-kas/recipes/enable.rb

+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+account_helper = AccountHelper.new(node)
+omnibus_helper = OmnibusHelper.new(node)
+
+working_dir = node['gitlab-kas']['dir']
+log_directory = node['gitlab-kas']['log_directory']
+env_directory = node['gitlab-kas']['env_directory']
+gitlab_kas_static_etc_dir = '/opt/gitlab/etc/gitlab-kas'
+gitlab_kas_config_file = File.join(working_dir, 'gitlab-kas-config.yml')
+gitlab_kas_authentication_secret_file = File.join(working_dir, 'authentication_secret_file')
+
+[
+  working_dir,
+  log_directory,
+  gitlab_kas_static_etc_dir
+].each do |dir|
+  directory dir do
+    owner account_helper.gitlab_user
+    mode '0700'
+    recursive true
+  end
+end
+
+version_file 'Create version file for Gitlab KAS' do
+  version_file_path File.join(working_dir, 'VERSION')
+  version_check_cmd '/opt/gitlab/embedded/bin/gitlab-kas --version'
+  notifies :restart, 'runit_service[gitlab-kas]'
+end
+
+file gitlab_kas_authentication_secret_file do
+  content node['gitlab-kas']['api_secret_key']
+  owner 'root'
+  group account_helper.gitlab_group
+  mode '0640'
+  notifies :restart, 'runit_service[gitlab-kas]'
+end
+
+template gitlab_kas_config_file do
+  source 'gitlab-kas-config.yml.erb'
+  owner 'root'
+  group account_helper.gitlab_group
+  mode '0640'
+  variables(
+    node['gitlab-kas'].to_hash.merge(
+      authentication_secret_file: gitlab_kas_authentication_secret_file
+    )
+  )
+  notifies :restart, 'runit_service[gitlab-kas]'
+end
+
+env_dir env_directory do
+  variables node['gitlab-kas']['env']
+  notifies :restart, 'runit_service[gitlab-kas]' if omnibus_helper.should_notify?('gitlab-kas')
+end
+
+runit_service 'gitlab-kas' do
+  options({
+    log_directory: log_directory,
+    env_directory: env_directory,
+    user: account_helper.gitlab_user,
+    groupname: account_helper.gitlab_group,
+    config_file: gitlab_kas_config_file,
+  }.merge(params))
+  log_options node['gitlab']['logging'].to_hash.merge(node['gitlab-kas'].to_hash)
+end

files/gitlab-cookbooks/gitlab-kas/templates/default/gitlab-kas-config.yml.erb

+agent:
+  configuration:
+    poll_period: <%= @agent_configuration_poll_period %>s
+  gitops:
+    poll_period: <%= @agent_gitops_poll_period %>s
+    project_info_cache_ttl: <%= @agent_gitops_project_info_cache_ttl %>s
+    project_info_cache_error_ttl: <%= @agent_gitops_project_info_cache_error_ttl %>s
+  info_cache_ttl: <%= @agent_info_cache_ttl %>s
+  info_cache_error_ttl: <%= @agent_info_cache_error_ttl %>s
+gitlab:
+  address: <%= @gitlab_address %>
+  authentication_secret_file: <%= @authentication_secret_file %>
+listen:
+  address: <%= @listen_address %>
+  network: <%= @listen_network %>
+  websocket: <%= @listen_websocket %>
+metrics:
+  usage_reporting_period: <%= @metrics_usage_reporting_period %>s

files/gitlab-cookbooks/gitlab-kas/templates/default/sv-gitlab-kas-log-config.erb

+<%= "s#@svlogd_size" if @svlogd_size %>
+<%= "n#@svlogd_num" if @svlogd_num %>
+<%= "t#@svlogd_timeout" if @svlogd_timeout %>
+<%= "!#@svlogd_filter" if @svlogd_filter %>
+<%= "u#@svlogd_udp" if @svlogd_udp %>
+<%= "p#@svlogd_prefix" if @svlogd_prefix %>

files/gitlab-cookbooks/gitlab-kas/templates/default/sv-gitlab-kas-log-run.erb

+#!/bin/sh
+exec svlogd -tt <%= @options[:log_directory] %>

files/gitlab-cookbooks/gitlab-kas/templates/default/sv-gitlab-kas-run.erb

+#!/bin/bash
+
+# Let runit capture all script error messages
+exec 2>&1
+
+<%= render('mount_point_check.erb', cookbook: 'gitlab') %>
+
+exec chpst -e <%= @options[:env_directory] %> -P \
+  -u <%= @options[:user] %>:<%= @options[:groupname] %> \
+  -U <%= @options[:user] %>:<%= @options[:groupname] %> \
+  /opt/gitlab/embedded/bin/gitlab-kas --configuration-file <%= @options[:config_file] %>
\ No newline at end of file

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -797,7 +797,7 @@ default['gitlab']['remote-syslog']['dir'] = "/var/opt/gitlab/remote-syslog"
 default['gitlab']['remote-syslog']['log_directory'] = "/var/log/gitlab/remote-syslog"
 default['gitlab']['remote-syslog']['destination_host'] = "localhost"
 default['gitlab']['remote-syslog']['destination_port'] = 514
-default['gitlab']['remote-syslog']['services'] = %w(redis nginx puma unicorn gitlab-rails gitlab-shell postgresql sidekiq gitlab-workhorse gitlab-pages praefect)
+default['gitlab']['remote-syslog']['services'] = %w(redis nginx puma unicorn gitlab-rails gitlab-shell postgresql sidekiq gitlab-workhorse gitlab-pages praefect gitlab-kas)
 ###
 # Logrotate
@@ -806,7 +806,7 @@ default['gitlab']['logrotate']['enable'] = false
 default['gitlab']['logrotate']['ha'] = false
 default['gitlab']['logrotate']['dir'] = "/var/opt/gitlab/logrotate"
 default['gitlab']['logrotate']['log_directory'] = "/var/log/gitlab/logrotate"
-default['gitlab']['logrotate']['services'] = %w(nginx puma actioncable unicorn gitlab-rails gitlab-shell gitlab-workhorse gitlab-pages)
+default['gitlab']['logrotate']['services'] = %w(nginx puma actioncable unicorn gitlab-rails gitlab-shell gitlab-workhorse gitlab-pages gitlab-kas)
 default['gitlab']['logrotate']['pre_sleep'] = 600 # sleep 10 minutes before rotating after start-up
 default['gitlab']['logrotate']['post_sleep'] = 3000 # wait 50 minutes after rotating

files/gitlab-cookbooks/gitlab/libraries/logrotate_helper.rb

@@ -33,6 +33,7 @@ class LogrotateHelper < AccountHelper
       'gitaly' => { username: gitlab_user, group: gitlab_group },
       'gitlab-exporter' => { username: gitlab_user, group: gitlab_group },
       'gitlab-pages' => { username: gitlab_user, group: gitlab_group },
+      'gitlab-kas' => { username: gitlab_user, group: gitlab_group },
       'gitlab-rails' => { username: gitlab_user, group: gitlab_group },
       'gitlab-shell' => { username: gitlab_user, group: gitlab_group },
       'gitlab-workhorse' => { username: gitlab_user, group: gitlab_group },

files/gitlab-cookbooks/gitlab/metadata.rb

@@ -18,5 +18,6 @@ depends 'mattermost'
 depends 'consul'
 depends 'gitaly'
 depends 'praefect'
+depends 'gitlab-kas'
 depends 'letsencrypt'
 depends 'nginx'