Merge branch...

Merge branch '2214-investigate-consolidating-attributes-postgresql-dir-and-postgresql-data_dir' into 'master'

First steps in consolidating attributes postgresql['dir'] and postgresql['data_dir']"

See merge request gitlab-org/omnibus-gitlab!2814

files/gitlab-cookbooks/gitlab-ee/attributes/default.rb

@@ -76,9 +76,9 @@ default['gitlab']['geo-postgresql'] = default['gitlab']['postgresql'].dup
 # We are inheriting default attributes from postgresql and changing below what should be different
 default['gitlab']['geo-postgresql']['enable'] = false
 default['gitlab']['geo-postgresql']['dir'] = '/var/opt/gitlab/geo-postgresql'
-default['gitlab']['geo-postgresql']['data_dir'] = '/var/opt/gitlab/geo-postgresql/data'
+default['gitlab']['geo-postgresql']['data_dir'] = nil
 default['gitlab']['geo-postgresql']['log_directory'] = '/var/log/gitlab/geo-postgresql'
-default['gitlab']['geo-postgresql']['unix_socket_directory'] = '/var/opt/gitlab/geo-postgresql'
+default['gitlab']['geo-postgresql']['unix_socket_directory'] = nil
 default['gitlab']['geo-postgresql']['ssl'] = 'off'
 # Postgres User's Environment Path
 default['gitlab']['geo-postgresql']['sql_user'] = 'gitlab_geo'

files/gitlab-cookbooks/gitlab-ee/recipes/geo-postgresql.rb

@@ -19,11 +19,11 @@ account_helper = AccountHelper.new(node)
 omnibus_helper = OmnibusHelper.new(node)
 gitlab_geo_helper = GitlabGeoHelper.new(node)
-postgresql_dir = node['gitlab']['geo-postgresql']['dir']
-postgresql_data_dir = node['gitlab']['geo-postgresql']['data_dir']
-postgresql_data_dir_symlink = File.join(postgresql_dir, 'data')
+node.default['gitlab']['geo-postgresql']['unix_socket_directory'] ||= node['gitlab']['geo-postgresql']['dir']
+node.default['gitlab']['geo-postgresql']['data_dir'] ||= "#{node['gitlab']['geo-postgresql']['dir']}/data"
+node.default['gitlab']['geo-postgresql']['home'] ||= node['gitlab']['geo-postgresql']['dir']
+
 postgresql_log_dir = node['gitlab']['geo-postgresql']['log_directory']
-postgresql_socket_dir = node['gitlab']['geo-postgresql']['unix_socket_directory']
 postgresql_username = account_helper.postgresql_user
 geo_pg_helper = GeoPgHelper.new(node)
@@ -31,14 +31,14 @@ fdw_helper = FdwHelper.new(node)
 include_recipe 'postgresql::user'
-directory postgresql_dir do
+directory node['gitlab']['geo-postgresql']['dir'] do
   owner postgresql_username
   mode '0755'
   recursive true
 end
 [
-  postgresql_data_dir,
+  node['gitlab']['geo-postgresql']['data_dir'],
   postgresql_log_dir
 ].each do |dir|
   directory dir do
@@ -48,18 +48,13 @@ end
   end
 end
-link postgresql_data_dir_symlink do
-  to postgresql_data_dir
-  not_if { postgresql_data_dir == postgresql_data_dir_symlink }
-end
-
-execute "/opt/gitlab/embedded/bin/initdb -D #{postgresql_data_dir} -E UTF8" do
+execute "/opt/gitlab/embedded/bin/initdb -D #{node['gitlab']['geo-postgresql']['data_dir']} -E UTF8" do
   user postgresql_username
   not_if { geo_pg_helper.bootstrapped? }
 end
-postgresql_config = File.join(postgresql_data_dir, 'postgresql.conf')
-postgresql_runtime_config = File.join(postgresql_data_dir, 'runtime.conf')
+postgresql_config = File.join(node['gitlab']['geo-postgresql']['data_dir'], 'postgresql.conf')
+postgresql_runtime_config = File.join(node['gitlab']['geo-postgresql']['data_dir'], 'runtime.conf')
 bootstrapping = !geo_pg_helper.bootstrapped?
 should_notify = omnibus_helper.should_notify?('geo-postgresql') && !bootstrapping
@@ -83,7 +78,7 @@ template postgresql_runtime_config do
   notifies :run, 'execute[reload geo-postgresql]', :immediately if should_notify
 end
-pg_hba_config = File.join(postgresql_data_dir, 'pg_hba.conf')
+pg_hba_config = File.join(node['gitlab']['geo-postgresql']['data_dir'], 'pg_hba.conf')
 template pg_hba_config do
   source 'pg_hba.conf.erb'
@@ -94,7 +89,7 @@ template pg_hba_config do
   notifies :restart, 'service[geo-postgresql]', :immediately if should_notify
 end
-template File.join(postgresql_data_dir, 'pg_ident.conf') do
+template File.join(node['gitlab']['geo-postgresql']['data_dir'], 'pg_ident.conf') do
   owner postgresql_username
   mode '0644'
   variables(node['gitlab']['geo-postgresql'].to_hash)
@@ -154,7 +149,7 @@ if node['gitlab']['geo-postgresql']['enable']
   postgresql_database geo_database_name do
     owner geo_pg_user
     database_port geo_pg_port
-    database_socket postgresql_socket_dir
+    database_socket node['gitlab']['geo-postgresql']['unix_socket_directory']
     helper geo_pg_helper
     action :create
   end

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -280,7 +280,7 @@ default['gitlab']['gitlab-rails']['db_username'] = "gitlab"
 default['gitlab']['gitlab-rails']['db_password'] = nil
 default['gitlab']['gitlab-rails']['db_load_balancing'] = { 'hosts' => [] }
 # Path to postgresql socket directory
-default['gitlab']['gitlab-rails']['db_host'] = "/var/opt/gitlab/postgresql"
+default['gitlab']['gitlab-rails']['db_host'] = nil
 default['gitlab']['gitlab-rails']['db_port'] = 5432
 default['gitlab']['gitlab-rails']['db_socket'] = nil
 default['gitlab']['gitlab-rails']['db_sslmode'] = nil
@@ -412,15 +412,15 @@ default['gitlab']['gitlab-shell']['git_data_directories'] = {
 default['gitlab']['postgresql']['enable'] = false
 default['gitlab']['postgresql']['ha'] = false
 default['gitlab']['postgresql']['dir'] = "/var/opt/gitlab/postgresql"
-default['gitlab']['postgresql']['data_dir'] = "/var/opt/gitlab/postgresql/data"
+default['gitlab']['postgresql']['data_dir'] = nil
 default['gitlab']['postgresql']['log_directory'] = "/var/log/gitlab/postgresql"
-default['gitlab']['postgresql']['unix_socket_directory'] = "/var/opt/gitlab/postgresql"
+default['gitlab']['postgresql']['unix_socket_directory'] = nil
 default['gitlab']['postgresql']['username'] = "gitlab-psql"
 default['gitlab']['postgresql']['group'] = "gitlab-psql"
 default['gitlab']['postgresql']['uid'] = nil
 default['gitlab']['postgresql']['gid'] = nil
 default['gitlab']['postgresql']['shell'] = "/bin/sh"
-default['gitlab']['postgresql']['home'] = "/var/opt/gitlab/postgresql"
+default['gitlab']['postgresql']['home'] = nil
 # Postgres User's Environment Path
 # defaults to /opt/gitlab/embedded/bin:/opt/gitlab/bin/$PATH. The install-dir path is set at build time
 default['gitlab']['postgresql']['user_path'] = "#{node['package']['install-dir']}/embedded/bin:#{node['package']['install-dir']}/bin:$PATH"
@@ -853,7 +853,6 @@ default['gitlab']['postgres-exporter']['home'] = '/var/opt/gitlab/postgres-expor
 default['gitlab']['postgres-exporter']['log_directory'] = "/var/log/gitlab/postgres-exporter"
 default['gitlab']['postgres-exporter']['listen_address'] = 'localhost:9187'
 default['gitlab']['postgres-exporter']['env'] = {
-  'DATA_SOURCE_NAME' => "user=#{node['gitlab']['postgresql']['username']} host=#{node['gitlab']['gitlab-rails']['db_host']} database=postgres sslmode=allow",
   'SSL_CERT_DIR' => "#{node['package']['install-dir']}/embedded/ssl/certs/"
 }

files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb

@@ -34,6 +34,7 @@ upgrade_status_dir = File.join(gitlab_rails_dir, "upgrade-status")
 # Set path to the private key used for communication between registry and Gitlab.
 node.default['gitlab']['gitlab-rails']['registry_key_path'] = File.join(gitlab_rails_etc_dir, "gitlab-registry.key")
+node.default['gitlab']['gitlab-rails']['db_host'] ||= node['gitlab']['postgresql']['dir']
 gitlab_user = account_helper.gitlab_user
 gitlab_group = account_helper.gitlab_group

files/gitlab-cookbooks/gitlab/recipes/postgres-exporter.rb

@@ -21,6 +21,10 @@ postgres_exporter_log_dir = node['gitlab']['postgres-exporter']['log_directory']
 postgres_exporter_static_etc_dir = "/opt/gitlab/etc/postgres-exporter"
 postgres_exporter_dir = node['gitlab']['postgres-exporter']['home']
+node.default['gitlab']['postgres-exporter']['env']['DATA_SOURCE_NAME'] = "user=#{node['gitlab']['postgresql']['username']} " \
+                                                                         "host=#{node['gitlab']['gitlab-rails']['db_host']} " \
+                                                                         "database=postgres sslmode=allow"
+
 include_recipe 'postgresql::user'
 directory postgres_exporter_log_dir do

files/gitlab-cookbooks/postgresql/recipes/bin.rb

@@ -17,7 +17,8 @@
 pg_helper = PgHelper.new(node)
 omnibus_helper = OmnibusHelper.new(node)
 postgresql_install_dir = File.join(node['package']['install-dir'], 'embedded/postgresql')
-postgresql_data_dir = node['gitlab']['postgresql']['data_dir']
+
+include_recipe 'postgresql::directory_locations'
 # This recipe will also be called standalone so the resource
 # won't exist for resource collection.
@@ -48,7 +49,7 @@ ruby_block "Link postgresql bin files to the correct version" do
     end
   end
   only_if do
-    !File.exist?(File.join(postgresql_data_dir, "PG_VERSION")) || pg_helper.version.major !~ /^#{pg_helper.database_version}/
+    !File.exist?(File.join(node['gitlab']['postgresql']['data_dir'], "PG_VERSION")) || pg_helper.version.major !~ /^#{pg_helper.database_version}/
   end
   notifies :restart, 'service[postgresql]', :immediately if omnibus_helper.should_notify?("postgresql") && resource_exists['service[postgresql]']
 end

files/gitlab-cookbooks/postgresql/recipes/directory_locations.rb

+node.default['gitlab']['postgresql']['unix_socket_directory'] ||= node['gitlab']['postgresql']['dir']
+node.default['gitlab']['postgresql']['data_dir'] ||= "#{node['gitlab']['postgresql']['dir']}/data"
+node.default['gitlab']['postgresql']['home'] ||= node['gitlab']['postgresql']['dir']

files/gitlab-cookbooks/postgresql/recipes/enable.rb

@@ -18,11 +18,9 @@
 account_helper = AccountHelper.new(node)
 omnibus_helper = OmnibusHelper.new(node)
-postgresql_dir = node['gitlab']['postgresql']['dir']
-postgresql_data_dir = node['gitlab']['postgresql']['data_dir']
-postgresql_data_dir_symlink = File.join(postgresql_dir, "data")
+include_recipe 'postgresql::directory_locations'
+
 postgresql_log_dir = node['gitlab']['postgresql']['log_directory']
-postgresql_socket_dir = node['gitlab']['postgresql']['unix_socket_directory']
 postgresql_username = account_helper.postgresql_user
 postgresql_group = account_helper.postgresql_group
@@ -30,14 +28,14 @@ pg_helper = PgHelper.new(node)
 include_recipe 'postgresql::user'
-directory postgresql_dir do
+directory node['gitlab']['postgresql']['dir'] do
   owner postgresql_username
   mode "0755"
   recursive true
 end
 [
-  postgresql_data_dir,
+  node['gitlab']['postgresql']['data_dir'],
   postgresql_log_dir
 ].each do |dir|
   directory dir do
@@ -47,11 +45,6 @@ end
   end
 end
-link postgresql_data_dir_symlink do
-  to postgresql_data_dir
-  not_if { postgresql_data_dir == postgresql_data_dir_symlink }
-end
-
 file File.join(node['gitlab']['postgresql']['home'], ".profile") do
   owner postgresql_username
   mode "0600"
@@ -78,16 +71,16 @@ sysctl "kernel.sem" do
   value sem
 end
-execute "/opt/gitlab/embedded/bin/initdb -D #{postgresql_data_dir} -E UTF8" do
+execute "/opt/gitlab/embedded/bin/initdb -D #{node['gitlab']['postgresql']['data_dir']} -E UTF8" do
   user postgresql_username
   not_if { pg_helper.bootstrapped? }
 end
 ##
-# Create SSL cert + key in the defined location. Paths are relative to postgresql_data_dir
+# Create SSL cert + key in the defined location. Paths are relative to node['gitlab']['postgresql']['data_dir']
 ##
-ssl_cert_file = File.absolute_path(node['gitlab']['postgresql']['ssl_cert_file'], postgresql_data_dir)
-ssl_key_file = File.absolute_path(node['gitlab']['postgresql']['ssl_key_file'], postgresql_data_dir)
+ssl_cert_file = File.absolute_path(node['gitlab']['postgresql']['ssl_cert_file'], node['gitlab']['postgresql']['data_dir'])
+ssl_key_file = File.absolute_path(node['gitlab']['postgresql']['ssl_key_file'], node['gitlab']['postgresql']['data_dir'])
 file ssl_cert_file do
   content node['gitlab']['postgresql']['internal_certificate']
@@ -107,8 +100,8 @@ file ssl_key_file do
   only_if { node['gitlab']['postgresql']['ssl'] == 'on' }
 end
-postgresql_config = File.join(postgresql_data_dir, "postgresql.conf")
-postgresql_runtime_config = File.join(postgresql_data_dir, 'runtime.conf')
+postgresql_config = File.join(node['gitlab']['postgresql']['data_dir'], "postgresql.conf")
+postgresql_runtime_config = File.join(node['gitlab']['postgresql']['data_dir'], 'runtime.conf')
 should_notify = omnibus_helper.should_notify?("postgresql")
 template postgresql_config do
@@ -131,7 +124,7 @@ template postgresql_runtime_config do
   notifies :run, 'execute[start postgresql]', :immediately if should_notify
 end
-pg_hba_config = File.join(postgresql_data_dir, "pg_hba.conf")
+pg_hba_config = File.join(node['gitlab']['postgresql']['data_dir'], "pg_hba.conf")
 template pg_hba_config do
   source 'pg_hba.conf.erb'
@@ -142,7 +135,7 @@ template pg_hba_config do
   notifies :run, 'execute[start postgresql]', :immediately if should_notify
 end
-template File.join(postgresql_data_dir, 'pg_ident.conf') do
+template File.join(node['gitlab']['postgresql']['data_dir'], 'pg_ident.conf') do
   owner postgresql_username
   mode "0644"
   variables(node['gitlab']['postgresql'].to_hash)
@@ -201,7 +194,7 @@ if node['gitlab']['gitlab-rails']['enable']
   end
   execute "create #{database_name} database" do
-    command "/opt/gitlab/embedded/bin/createdb --port #{pg_port} -h #{postgresql_socket_dir} -O #{gitlab_sql_user} #{database_name}"
+    command "/opt/gitlab/embedded/bin/createdb --port #{pg_port} -h #{node['gitlab']['postgresql']['unix_socket_directory']} -O #{gitlab_sql_user} #{database_name}"
     user postgresql_username
     retries 30
     not_if { !pg_helper.is_running? || pg_helper.database_exists?(database_name) || pg_helper.is_slave? }

spec/chef/postgresql/resources/database_spec.rb

 require 'chef_helper'
 describe 'postgresql_database' do
-  let(:runner) { ChefSpec::SoloRunner.new(step_into: %w(postgresql_database)) }
+  let(:runner) do
+    ChefSpec::SoloRunner.new(step_into: %w(postgresql_database)) do |node|
+      # unix_socket_directory is normally conditionally set in postgresql::enable
+      # which is not executed as part of this spec
+      node.normal['gitlab']['postgresql']['unix_socket_directory'] = '/var/opt/gitlab/postgresql'
+    end
+  end
   context 'create' do
     let(:chef_run) { runner.converge('test_gitlab_postgresql_database::create') }