Merge branch 'master' of gitlab.com:gitlab-org/omnibus-gitlab into pitr

doc/docker/README.md

@@ -341,7 +341,7 @@ the GitLab version you want to run, for example `gitlab/gitlab-ee:12.1.3-ce.0`.
 You can make Docker to use your IP address and forward all traffic to the
 GitLab container by modifying the `--publish` flag.
-To expose GitLab CE on IP `198.51.100.1`:
+To expose GitLab on IP `198.51.100.1`:
 ```shell
 sudo docker run --detach \
@@ -452,7 +452,7 @@ To update GitLab that was [installed using Docker Engine](#install-gitlab-using-
    sudo docker rm gitlab
    ```
-1. Pull the new image. For example, the latest GitLab CE image:
+1. Pull the new image. For example, the latest GitLab image:
    ```shell
    sudo docker pull gitlab/gitlab-ee:latest
@@ -600,3 +600,30 @@ sudo setfacl -mR default:group:docker:rwx $GITLAB_HOME
 The default group is `docker`. If you changed the group, be sure to update your
 commands.
+
+### /dev/shm mount not having enough space in Docker container
+
+GitLab comes with a Prometheus metrics endpoint at `/-/metrics` to expose a
+variety of statistics on the health and performance of GitLab. The files
+required for this gets written to a temporary file system (like `/run` or
+`/dev/shm`).
+
+By default, Docker allocates 64Mb to the shared memory directory (mounted at
+`/dev/shm`). This is insufficient to hold all the Prometheus metrics related
+files generated, and will generate error logs like the following:
+
+```plaintext
+writing value to /dev/shm/gitlab/sidekiq/gauge_all_sidekiq_0-1.db failed with unmapped file
+writing value to /dev/shm/gitlab/sidekiq/gauge_all_sidekiq_0-1.db failed with unmapped file
+writing value to /dev/shm/gitlab/sidekiq/gauge_all_sidekiq_0-1.db failed with unmapped file
+writing value to /dev/shm/gitlab/sidekiq/histogram_sidekiq_0-0.db failed with unmapped file
+writing value to /dev/shm/gitlab/sidekiq/histogram_sidekiq_0-0.db failed with unmapped file
+writing value to /dev/shm/gitlab/sidekiq/histogram_sidekiq_0-0.db failed with unmapped file
+writing value to /dev/shm/gitlab/sidekiq/histogram_sidekiq_0-0.db failed with unmapped file
+```
+
+Other than disabling the Prometheus Metrics from the Admin page, the recommended
+solution to fix this problem is to increase the size of shm to at least 256Mb.
+If using `docker run`, this can be done by passing the flag `--shm-size 256m`.
+If using a `docker-compose.yml` file, the `shm_size` key can be used for this
+purpose.

docker/assets/download-package

@@ -9,3 +9,9 @@ else
     # to authenticate to access it. Hence, we pass PRIVATE-TOKEN header.
     wget --quiet --header "PRIVATE-TOKEN: ${TRIGGER_PRIVATE_TOKEN}" ${DOWNLOAD_URL} -O /tmp/gitlab.deb
 fi
+
+results=$?
+if [ ${results} -ne 0 ]; then
+    >&2 echo "There was an error downloading ${DOWNLOAD_URL}. Please check the output for more information"
+    exit ${results}
+fi

files/gitlab-config-template/gitlab.rb.template

@@ -73,8 +73,26 @@ external_url 'GENERATED_EXTERNAL_URL'
 ###! By default, we'll allow 95% of the the worker timeout
 # gitlab_rails['max_request_duration_seconds'] = 57
+### GitLab email server settings
+###! Docs: https://docs.gitlab.com/omnibus/settings/smtp.html
+###! **Use smtp instead of sendmail/postfix.**
+
+# gitlab_rails['smtp_enable'] = true
+# gitlab_rails['smtp_address'] = "smtp.server"
+# gitlab_rails['smtp_port'] = 465
+# gitlab_rails['smtp_user_name'] = "smtp user"
+# gitlab_rails['smtp_password'] = "smtp password"
+# gitlab_rails['smtp_domain'] = "example.com"
+# gitlab_rails['smtp_authentication'] = "login"
+# gitlab_rails['smtp_enable_starttls_auto'] = true
+# gitlab_rails['smtp_tls'] = false
+
 ### Email Settings
+
 # gitlab_rails['gitlab_email_enabled'] = true
+
+##! If your SMTP server does not like the default 'From: gitlab@gitlab.example.com'
+##! can change the 'From' with this setting.
 # gitlab_rails['gitlab_email_from'] = 'example@example.com'
 # gitlab_rails['gitlab_email_display_name'] = 'Example'
 # gitlab_rails['gitlab_email_reply_to'] = 'noreply@example.com'
@@ -271,6 +289,7 @@ gitlab_rails['object_store']['objects']['uploads']['bucket'] = nil
 gitlab_rails['object_store']['objects']['packages']['bucket'] = nil
 gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = nil
 gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = nil
+gitlab_rails['object_store']['objects']['pages']['bucket'] = nil
 ### Job Artifacts
 # gitlab_rails['artifacts_enabled'] = true
@@ -374,6 +393,21 @@ gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = nil
 #   # 'path_style' => false # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
 # }
+### GitLab Pages
+# gitlab_rails['pages_object_store_enabled'] = false
+# gitlab_rails['pages_object_store_remote_directory'] = "pages"
+# gitlab_rails['pages_object_store_connection'] = {
+#   'provider' => 'AWS',
+#   'region' => 'eu-west-1',
+#   'aws_access_key_id' => 'AWS_ACCESS_KEY_ID',
+#   'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY',
+#   # # The below options configure an S3 compatible host instead of AWS
+#   # 'host' => 's3.amazonaws.com',
+#   # 'aws_signature_version' => 4, # For creation of signed URLs. Set to 2 if provider does not support v4.
+#   # 'endpoint' => 'https://s3.amazonaws.com', # default: nil - Useful for S3 compliant services such as DigitalOcean Spaces
+#   # 'path_style' => false # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
+# }
+
 ### Impersonation settings
 # gitlab_rails['impersonation_enabled'] = true
@@ -655,20 +689,6 @@ gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = nil
 # gitlab_rails['redis_actioncable_instance'] = nil
 # gitlab_rails['redis_actioncable_sentinels'] = nil
-### GitLab email server settings
-###! Docs: https://docs.gitlab.com/omnibus/settings/smtp.html
-###! **Use smtp instead of sendmail/postfix.**
-
-# gitlab_rails['smtp_enable'] = true
-# gitlab_rails['smtp_address'] = "smtp.server"
-# gitlab_rails['smtp_port'] = 465
-# gitlab_rails['smtp_user_name'] = "smtp user"
-# gitlab_rails['smtp_password'] = "smtp password"
-# gitlab_rails['smtp_domain'] = "example.com"
-# gitlab_rails['smtp_authentication'] = "login"
-# gitlab_rails['smtp_enable_starttls_auto'] = true
-# gitlab_rails['smtp_tls'] = false
-
 ###! **Can be: 'none', 'peer', 'client_once', 'fail_if_no_peer_cert'**
 ###! Docs: http://api.rubyonrails.org/classes/ActionMailer/Base.html
 # gitlab_rails['smtp_openssl_verify_mode'] = 'none'
@@ -1032,6 +1052,7 @@ gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = nil
 ### Replication settings
 ###! Note, some replication settings do not require a full restart. They are documented below.
 # postgresql['wal_level'] = "hot_standby"
+# postgresql['wal_log_hints'] = 'off'
 # postgresql['max_wal_senders'] = 5
 # postgresql['max_replication_slots'] = 0
 # postgresql['max_locks_per_transaction'] = 128

files/gitlab-cookbooks/consul/resources/consul_service.rb

 resource_name :consul_service
+provides :consul_service
 property :service_name, String, name_property: true
 property :ip_address, [String, nil], default: nil

files/gitlab-cookbooks/gitlab-ee/attributes/default.rb

@@ -119,6 +119,7 @@ default['gitlab']['geo-postgresql']['default_statistics_target'] = 1000
 # Replication settings
 default['gitlab']['geo-postgresql']['wal_level'] = 'minimal'
+default['gitlab']['geo-postgresql']['wal_log_hints'] = 'off'
 default['gitlab']['geo-postgresql']['max_wal_senders'] = 0
 default['gitlab']['geo-postgresql']['wal_keep_segments'] = 10
 default['gitlab']['geo-postgresql']['hot_standby'] = 'off'

files/gitlab-cookbooks/gitlab-ee/resources/pgbouncer_user.rb

 resource_name :pgbouncer_user
+provides :pgbouncer_user
 property :type, String, name_property: true
 property :account_helper, default: AccountHelper.new(node)

files/gitlab-cookbooks/gitlab-ee/resources/postgresql_fdw.rb

 resource_name :postgresql_fdw
+provides :postgresql_fdw
 property :server_name, String, name_property: true
 property :db_name, String

files/gitlab-cookbooks/gitlab-ee/resources/sentinel_service.rb

 resource_name :sentinel_service
+provides :sentinel_service
 property :config_path, String
 property :redis_configuration, Hash

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -187,6 +187,8 @@ default['gitlab']['gitlab-rails']['object_store']['objects']['dependency_proxy']
 default['gitlab']['gitlab-rails']['object_store']['objects']['dependency_proxy']['bucket'] = nil
 default['gitlab']['gitlab-rails']['object_store']['objects']['terraform_state'] = {}
 default['gitlab']['gitlab-rails']['object_store']['objects']['terraform_state']['bucket'] = nil
+default['gitlab']['gitlab-rails']['object_store']['objects']['pages'] = {}
+default['gitlab']['gitlab-rails']['object_store']['objects']['pages']['bucket'] = nil
 default['gitlab']['gitlab-rails']['artifacts_enabled'] = true
 default['gitlab']['gitlab-rails']['artifacts_path'] = nil
@@ -262,6 +264,9 @@ default['gitlab']['gitlab-rails']['pages_host'] = nil
 default['gitlab']['gitlab-rails']['pages_port'] = nil
 default['gitlab']['gitlab-rails']['pages_https'] = false
 default['gitlab']['gitlab-rails']['pages_path'] = nil
+default['gitlab']['gitlab-rails']['pages_object_store_enabled'] = false
+default['gitlab']['gitlab-rails']['pages_object_store_remote_directory'] = 'pages'
+default['gitlab']['gitlab-rails']['pages_object_store_connection'] = {}
 default['gitlab']['gitlab-rails']['registry_enabled'] = false
 default['gitlab']['gitlab-rails']['registry_host'] = nil
 default['gitlab']['gitlab-rails']['registry_port'] = nil

files/gitlab-cookbooks/gitlab/libraries/puma.rb

@@ -19,6 +19,7 @@ module Puma
   class << self
     def parse_variables
       only_one_allowed!
+      parse_listen_address
     end
     def only_one_allowed!
@@ -27,6 +28,11 @@ module Puma
       raise 'Only one web server (Puma or Unicorn) can be enabled at the same time!'
     end
+    def parse_listen_address
+      puma_socket = Gitlab['puma']['socket'] || Gitlab['node']['gitlab']['puma']['socket']
+      Gitlab['gitlab_workhorse']['auth_socket'] = puma_socket if Gitlab['gitlab_workhorse']['auth_backend'].nil?
+    end
+
     def workers(total_memory = Gitlab['node']['memory']['total'].to_i)
       [
         2, # Two is the minimum or web editor will no longer work.

files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb

@@ -298,6 +298,9 @@ templatesymlink "Create a gitlab.yml and create a symlink to Rails root" do
       pages_external_https: node['gitlab']['gitlab-pages']['external_https'],
       pages_artifacts_server: node['gitlab']['gitlab-pages']['artifacts_server'],
       pages_access_control: node['gitlab']['gitlab-pages']['access_control'],
+      pages_object_store_enabled: node['gitlab']['gitlab-rails']['pages_object_store_enabled'],
+      pages_object_store_remote_directory: node['gitlab']['gitlab-rails']['pages_object_store_remote_directory'],
+      pages_object_store_connection: node['gitlab']['gitlab-rails']['pages_object_store_connection'],
       mattermost_host: mattermost_host,
       mattermost_enabled: node['mattermost']['enable'] || !mattermost_host.nil?,
       sidekiq: node['gitlab']['sidekiq'],

files/gitlab-cookbooks/gitlab/resources/database_objects.rb

 resource_name :database_objects
+provides :database_objects
 property :pg_helper, [GeoPgHelper, PgHelper], required: true
 property :account_helper, [AccountHelper], required: true

files/gitlab-cookbooks/gitlab/resources/puma_config.rb

 resource_name :puma_config
+provides :puma_config
 property :filename, String, name_property: true
 property :tag, String, default: 'gitlab-puma-worker'

files/gitlab-cookbooks/gitlab/resources/sidekiq_service.rb

 resource_name :sidekiq_service
+provides :sidekiq_service
 property :rails_app, String, default: 'gitlab-rails'
 property :user, default: lazy { node['gitlab']['user']['username'] }

files/gitlab-cookbooks/gitlab/resources/unicorn_config.rb

 resource_name :unicorn_config
+provides :unicorn_config
 property :listen, Hash
 property :working_directory

files/gitlab-cookbooks/gitlab/resources/unicorn_service.rb

 resource_name :unicorn_service
+provides :unicorn_service
 property :rails_app
 property :svc, String, name_property: true

files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb

@@ -336,6 +336,10 @@ production: &base
     external_http: <%= @pages_external_http.to_json %>
     external_https: <%= @pages_external_https.to_json %>
     artifacts_server: <%= @pages_artifacts_server %>
+    object_store:
+      enabled: <%= @pages_object_store_enabled %>
+      remote_directory: <%= quote(@pages_object_store_remote_directory) %>
+      connection: <%= @pages_object_store_connection.to_json %>
   ## Gravatar
   ## For Libravatar see: https://docs.gitlab.com/ee/customization/libravatar.html

files/gitlab-cookbooks/package/libraries/helpers/gitlab_cluster_helper.rb

+require 'chef/json_compat'
+require 'chef/log'
+require_relative 'logging_helper'
+
+class GitlabClusterHelper
+  CONFIG_PATH = '/etc/gitlab'.freeze
+  JSON_FILE   = '/etc/gitlab/gitlab-cluster.json'.freeze
+
+  class << self
+    def config_available?
+      File.exist?(JSON_FILE)
+    end
+  end
+
+  def config
+    return @config if defined?(@config)
+
+    @config = load_from_file
+  end
+
+  # Roles defined in the JSON file overrides roles from /etc/gitlab/gitlab.rb
+  def load_roles!
+    load_role!('geo_primary_role', 'primary')
+    load_role!('geo_secondary_role', 'secondary')
+  end
+
+  # Write configuration to the local JSON file overriding current settings
+  def write_to_file!
+    return unless File.directory?(CONFIG_PATH)
+
+    json_config = Chef::JSONCompat.to_json_pretty(config)
+
+    File.open(JSON_FILE, 'w', 0600) do |f|
+      f.puts(json_config)
+      f.chmod(0600)
+    end
+  end
+
+  private
+
+  # Load configuration from the local JSON file
+  def load_from_file
+    return {} unless self.class.config_available?
+
+    Chef::JSONCompat.from_json(File.read(JSON_FILE))
+  end
+
+  def load_role!(role, key)
+    return unless config.key?(key)
+
+    print_warning(role, key) if Gitlab[role]['enable']
+    Gitlab[role]['enable'] = config[key]
+  end
+
+  def print_warning(role, key)
+    LoggingHelper.warning "The #{role} is defined in #{JSON_FILE} as #{key} and takes priority over the role in the /etc/gitlab/gitlab.rb"
+  end
+end

files/gitlab-cookbooks/package/libraries/helpers/settings_helper.rb

@@ -22,6 +22,7 @@ require 'securerandom'
 require 'uri'
 require_relative '../config_mash.rb'
+require_relative 'gitlab_cluster_helper'
 module SettingsHelper
   def self.extended(base)
@@ -153,6 +154,9 @@ module SettingsHelper
     Services.enable_group(Services::SYSTEM_GROUP)
     RolesHelper.parse_enabled
+    # Roles defined in the cluster configuration file overrides roles from /etc/gitlab/gitlab.rb
+    gitlab_cluster_helper.load_roles!
+
     # Load our roles
     DefaultRole.load_role
     @available_roles.each do |key, value|
@@ -203,6 +207,10 @@ module SettingsHelper
   private
+  def gitlab_cluster_helper
+    @gitlab_cluster_helper ||= GitlabClusterHelper.new
+  end
+
   # Sort settings by their sequence value
   def sorted_settings
     @settings.select { |_k, value| !value[:ee] || Gitlab['edition'] == :ee }.sort_by { |_k, value| value[:priority] }

files/gitlab-cookbooks/package/libraries/omnibus_helper.rb

@@ -104,7 +104,8 @@ class OmnibusHelper
     {
       'raspbian-9' => 'GitLab 13.4',
       'debian-8' => 'GitLab 13.4',
-      'centos-6' => 'GitLab 13.7'
+      'centos-6' => 'GitLab 13.7',
+      'opensuseleap-15.1' => 'GitLab 13.7'
     }
   end