Merge branch '1626-provide-a-way-to-upgrade-bundled-postgresql' into 'master'

Add ability to update omnibus provided PostgreSQL to 9.6.1

* Installs postgresql to version specific directory, and symlinks binaries
into /opt/gitlab/embedded/bin
* Adds pg-upgrade option to gitlab-ctl to upgrade the database application and data
* Provides revert-pg-upgrade option to revert the database in case of issue

See merge request !1058

config/projects/gitlab.rb

@@ -70,7 +70,6 @@ override :ruby, version: '2.3.1', source: { md5: '0d896c2e7fd54f722b399f407e48a4
 override :rubygems, version: '2.6.6'
 override :'chef-gem', version: '12.12.15'
 override :redis, version: '3.2.5', source: { md5: 'd3d2b4dd4b2a3e07ee6f63c526b66b08' }
-override :postgresql, version: '9.2.18', source: { md5: 'fd175eb5f29557c6ef2eeaf340330f9a' }
 override :liblzma, version: '5.2.2', source: { md5: '7cf6a8544a7dae8e8106fdf7addfa28c' }
 override :libxml2, version: '2.9.4', source: { md5: 'ae249165c173b1ff386ee8ad676815f5' }
 override :pcre, version: '8.38', source: { md5: '8a353fe1450216b6655dfcf3561716d9', url: "http://downloads.sourceforge.net/project/pcre/pcre/8.38/pcre-8.38.tar.gz" }

config/software/gitlab-rails.rb

@@ -42,6 +42,7 @@ dependency "curl"
 dependency "rsync"
 dependency "libicu"
 dependency "postgresql"
+dependency "postgresql_new"
 dependency "python-docutils"
 dependency "krb5"
 dependency "registry"

config/software/postgresql.rb

+#
+# Copyright 2012-2014 Chef Software, Inc.
+# Copyright:: Copyright (c) 2016 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name "postgresql"
+default_version '9.2.18'
+
+license "PostgreSQL"
+license_file "COPYRIGHT"
+
+dependency "zlib"
+dependency "openssl"
+dependency "libedit"
+dependency "ncurses"
+dependency "libossp-uuid"
+dependency "config_guess"
+
+version '9.2.18' do
+  source md5: 'fd175eb5f29557c6ef2eeaf340330f9a'
+end
+
+source url: "https://ftp.postgresql.org/pub/source/v#{version}/postgresql-#{version}.tar.bz2"
+
+relative_path "postgresql-#{version}"
+
+build do
+  env = with_standard_compiler_flags(with_embedded_path)
+  prefix = "#{install_dir}/embedded/postgresql/#{version}"
+  update_config_guess(target: "config")
+
+  command "./configure" \
+    " --prefix=#{prefix}" \
+    " --with-libedit-preferred" \
+    " --with-openssl" \
+    " --with-ossp-uuid", env: env
+
+  make "world -j #{workers}", env: env
+  make "install-world", env: env
+
+  block 'link bin files' do
+    Dir.glob("#{prefix}/bin/*").each do |bin_file|
+      link bin_file, "#{install_dir}/embedded/bin/#{File.basename(bin_file)}"
+    end
+  end
+end

config/software/postgresql_new.rb

+#
+# Copyright 2012-2014 Chef Software, Inc.
+# Copyright:: Copyright (c) 2016 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name "postgresql_new"
+default_version "9.6.1"
+
+license "PostgreSQL"
+license_file "COPYRIGHT"
+
+dependency "zlib"
+dependency "openssl"
+dependency "libedit"
+dependency "ncurses"
+dependency "libossp-uuid"
+dependency "config_guess"
+
+version "9.6.0" do
+  source md5: "c5af6ebb790ab877e1d2e56e19cebb29"
+end
+
+version '9.6.1' do
+  source sha256: 'e5101e0a49141fc12a7018c6dad594694d3a3325f5ab71e93e0e51bd94e51fcd'
+end
+
+source url: "https://ftp.postgresql.org/pub/source/v#{version}/postgresql-#{version}.tar.bz2"
+
+relative_path "postgresql-#{version}"
+
+build do
+  env = with_standard_compiler_flags(with_embedded_path)
+  prefix = "#{install_dir}/embedded/postgresql/#{version}"
+  update_config_guess(target: "config")
+
+  command "./configure" \
+          " --prefix=#{prefix}" \
+          " --with-libedit-preferred" \
+          " --with-openssl" \
+          " --with-ossp-uuid", env: env
+
+  make "world -j #{workers}", env: env
+  make "install-world", env: env
+end

doc/maintenance/README.md

@@ -187,3 +187,46 @@ registry['storage'] = {
 ```
 and run `sudo gitlab-ctl reconfigure`.
+
+#### Upgrade postgresql database
+
+Currently GitLab Omnibus runs PostgreSQL 9.2.18 by default. Version 9.6.1 is included as an option for users to manually upgrade. The next major release will ship with a newer PostgresQL by default, and will upgrade existing omnibus installations when they are upgraded.
+
+In order to be able to manually upgrade, please check the folowing:
+* You're currently running the latest version of GitLab and it is working. If you recently upgraded, make sure that `gitlab-ctl reconfigure` has successfully run before you proceed.
+* You're using the bundled version of PostgreSQL. Look for `postgresql['enable']` to be `true`, commented out, or absent from `/etc/gitlab/gitlab.rb`
+* You haven't already upgraded. Running `/opt/gitlab/embedded/bin/psql --version` should print `psql (PostgreSQL) 9.2.18`
+* You will need to have sufficient disk space for two copies of your database. Do not attempt to upgrade unless you have enough free space available. If the partition where the database resides does not have enough space (default location is `/var/opt/gitlab/postgresql/data`), you can pass the argument `--tmp-dir $DIR` to the command.
+
+Please note:
+* This upgrade does require downtime as the database must be down while the upgrade is being performed. The length of time entirely depends on the size of your database.
+
+To perform the ugprade, run the command:
+
+```
+sudo gitlab-ctl pg-upgrade
+```
+This command performs the following steps:
+1. Checks to ensure the database is in a known good state
+1. Shuts down the existing database
+1. Changes the symlinks in `/opt/gitlab/embedded/bin/` for PostgreSQL to point to the newer version of the database
+1. Creates a new directory containing a new, empty database with a locale matching the existing database
+1. Uses the `pg_upgrade` tool to copy the data from the old database to the new database
+1. Moves the old database out of the way
+1. Moves the new database to the expected location
+1. Calls `gitlab-ctl` reconfigure to make any needed changes, and start the new database server.
+1. If any errors are detected during this process, it should immediately revert to the old version of the database.
+
+Once this step is complete, verify everything is working as expected. If so, you can remove the old database with:
+
+```
+sudo rm -rf /var/opt/gitlab/postgresql/data.9.2.18
+```
+
+If you run into an issue, and wish to downgrade the version of PostgreSQL, run:
+
+```
+sudo gitlab-ctl revert-pg-upgrade
+```
+Please note:
+This will revert your database and data to what was there before you upgraded the database. Any changes you have made since the ugprade will be lost.

files/gitlab-config-template/gitlab.rb.template

@@ -536,6 +536,10 @@ external_url 'GENERATED_EXTERNAL_URL'
 # postgresql['wal_keep_segments'] = 10
 # postgresql['hot_standby'] = "off"
+## Available in PostgreSQL 9.6 and later
+# postgresql['min_wal_size'] = 80MB
+# postgresql['max_wal_size'] = 1GB
+
 ################
 # GitLab Redis #
 ################

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -379,6 +379,8 @@ default['gitlab']['postgresql']['maintenance_work_mem'] = "16MB"
 default['gitlab']['postgresql']['effective_cache_size'] = "#{(node['memory']['total'].to_i / 2) / (1024)}MB"
 default['gitlab']['postgresql']['log_min_duration_statement'] = -1 # Disable slow query logging by default
 default['gitlab']['postgresql']['checkpoint_segments'] = 10
+default['gitlab']['postgresql']['min_wal_size'] = '80MB'
+default['gitlab']['postgresql']['max_wal_size'] = '1GB'
 default['gitlab']['postgresql']['checkpoint_timeout'] = "5min"
 default['gitlab']['postgresql']['checkpoint_completion_target'] = 0.9
 default['gitlab']['postgresql']['checkpoint_warning'] = "30s"

files/gitlab-cookbooks/gitlab/libraries/helper.rb

@@ -79,6 +79,10 @@ class PgHelper
     cmd = ["/opt/gitlab/bin/gitlab-psql", cmd_list.join(" ")].join(" ")
     success?(cmd)
   end
+
+  def version
+    VersionHelper.version('/opt/gitlab/embedded/bin/psql --version').split.last
+  end
 end
 class OmnibusHelper

files/gitlab-cookbooks/gitlab/recipes/postgresql.rb

@@ -24,6 +24,8 @@ postgresql_log_dir = node['gitlab']['postgresql']['log_directory']
 postgresql_socket_dir = node['gitlab']['postgresql']['unix_socket_directory']
 postgresql_user = account_helper.postgresgl_user
+pg_helper = PgHelper.new(node)
+
 account "Postgresql user and group" do
   username postgresql_user
   uid node['gitlab']['postgresql']['uid']
@@ -88,6 +90,8 @@ end
 postgresql_config = File.join(postgresql_data_dir, "postgresql.conf")
+node.default['gitlab']['postgresql']['version'] = pg_helper.version
+
 template postgresql_config do
   source "postgresql.conf.erb"
   owner postgresql_user
@@ -141,7 +145,6 @@ template "/opt/gitlab/etc/gitlab-psql-rc" do
   group 'root'
 end
-pg_helper = PgHelper.new(node)
 pg_port = node['gitlab']['postgresql']['port']
 database_name = node['gitlab']['gitlab-rails']['db_database']
 gitlab_sql_user = node['gitlab']['postgresql']['sql_user']

files/gitlab-cookbooks/gitlab/templates/default/postgresql.conf.erb

@@ -69,7 +69,11 @@ max_connections = <%= node['gitlab']['postgresql']['max_connections'] %>      #
 # Note:  Increasing max_connections costs ~400 bytes of shared memory per
 # connection slot, plus lock space (see max_locks_per_transaction).
 #superuser_reserved_connections = 3 # (change requires restart)
+<% if node['gitlab']['postgresql']['version'] == "9.2.18" %>
 unix_socket_directory = '<%= node['gitlab']['postgresql']['unix_socket_directory'] %>'   # (change requires restart)
+<% else %>
+unix_socket_directories = '<%= node['gitlab']['postgresql']['unix_socket_directory'] %>'   # (change requires restart)
+<% end %>
 #unix_socket_group = ''     # (change requires restart)
 #unix_socket_permissions = 0777   # begin with 0 to use octal notation
           # (change requires restart)
@@ -174,8 +178,12 @@ wal_buffers = <%= node['gitlab']['postgresql']['wal_buffers'] %> # -1     # min
 #commit_siblings = 5      # range 1-1000
 # - Checkpoints -
-
+<% if node['gitlab']['postgresql']['version'] == "9.2.18" %>
 checkpoint_segments = <%= node['gitlab']['postgresql']['checkpoint_segments'] %>		# in logfile segments, min 1, 16MB each, default 3
+<% else %>
+min_wal_size = <%= node['gitlab']['postgresql']['min_wal_size'] %>
+max_wal_size = <%= node['gitlab']['postgresql']['max_wal_size'] %>
+<% end %>
 checkpoint_timeout = <%= node['gitlab']['postgresql']['checkpoint_timeout'] %>		# range 30s-1h, default 5min
 checkpoint_completion_target = <%= node['gitlab']['postgresql']['checkpoint_completion_target'] %>	# checkpoint target duration, 0.0 - 1.0, default 0.5
 checkpoint_warning = <%= node['gitlab']['postgresql']['checkpoint_warning'] %>		# 0 disables, default 30s

files/gitlab-ctl-commands/upgradedb.rb

+#
+# Copyright:: Copyright (c) 2016 GitLab Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'mixlib/shellout'
+require 'optparse'
+
+options = {}
+
+OptionParser.new do |opts|
+  opts.on('-tDIR', '--tmp-dir=DIR', 'Storage location for temporary data') do |t|
+    options[:tmp_dir] = t
+  end
+end.parse!(ARGV)
+
+DATA_DIR = "#{data_path}/postgresql/data".freeze
+INST_DIR = "#{base_path}/embedded/postgresql".freeze
+TMP_DATA_DIR = options.key?(:tmp_dir) ? "#{options[:tmp_dir]}/data" : DATA_DIR
+
+add_command_under_category 'revert-pg-upgrade', 'database',
+                           'Run this to revert to the previous version of the database',
+                           1 do |_cmd_name|
+  maintenance_mode('enable')
+  if progress_message('Checking if we need to downgrade') do
+    fetch_running_version == default_version
+  end
+    log "Already running #{default_version}"
+    exit! 1
+  end
+
+  unless Dir.exist?("#{TMP_DATA_DIR}.#{default_version}")
+    log "#{TMP_DATA_DIR}.#{default_version} does not exist, cannot revert data"
+    log 'Will proceed with reverting the running program version only, unless you interrupt'
+  end
+
+  log "Reverting database to #{default_version} in 5 seconds"
+  log '=== WARNING ==='
+  log 'This will revert the database to what it was before you upgraded, including the data.'
+  log "Please hit Ctrl-C now if this isn't what you were looking for"
+  log '=== WARNING ==='
+  begin
+    sleep 5
+  rescue Interrupt
+    log 'Received interrupt, not doing anything'
+    exit! 0
+  end
+  revert
+  maintenance_mode('disable')
+end
+
+add_command_under_category 'pg-upgrade', 'database',
+                           'Upgrade the PostgreSQL DB to the latest supported version',
+                           1 do |_cmd_name|
+  running_version = fetch_running_version
+
+  unless progress_message(
+    'Checking for an omnibus managed postgresql') do
+      !running_version.nil? && \
+      get_all_services.member?('postgresql')
+    end
+    $stderr.puts 'No currently installed postgresql in the omnibus instance found.'
+    exit! 1
+  end
+
+  unless progress_message('Checking version of running PostgreSQL') do
+    running_version == default_version
+  end
+    log "psql reports #{running_version}, we're expecting " \
+    "#{default_version}, cannot proceed"
+    exit! 1
+  end
+
+  if progress_message(
+    'Checking for a newer version of PostgreSQL to install') do
+      upgrade_version.nil?
+      Dir.exist?("#{INST_DIR}/#{upgrade_version}")
+    end
+    log "Upgrading PostgreSQL to #{upgrade_version}"
+  else
+    $stderr.puts 'No new version of PostgreSQL installed, nothing to upgrade to'
+    exit! 1
+  end
+
+  unless progress_message('Checking if existing PostgreSQL instances needs to be upgraded') do
+    running_version != upgrade_version
+  end
+    log "Already at #{upgrade_version}, nothing to do"
+    exit! 0
+  end
+
+  # In case someone altered the db outside of the recommended parameters,
+  # make sure everything is as we expect
+  unless progress_message(
+    'Checking if the PostgreSQL directory has been symlinked elsewhere'
+  ) do
+    !File.symlink?(DATA_DIR)
+  end
+    log "#{DATA_DIR} is a symlink to another directory. Will not proceed"
+    exit! 1
+  end
+
+  unless progress_message(
+    'Checking if PostgreSQL bin files are symlinked to the expected location'
+  ) do
+    Dir.glob("#{INST_DIR}/#{default_version}/bin/*").each do |bin_file|
+      link = "#{base_path}/embedded/bin/#{File.basename(bin_file)}"
+      File.symlink?(link) && File.readlink(link).eql?(bin_file)
+    end
+  end
+    log "#{link} is not linked to #{bin_file}, unable to proceed with non-standard installation"
+    exit! 1
+  end
+
+  # All tests have passed, this should be an upgradable instance.
+  maintenance_mode('enable')
+
+  # Get the existing locale before we move on
+  locale, encoding = fetch_lc_collate.strip.split('.')
+  progress_message('Stopping the database') do
+    run_sv_command_for_service('stop', 'postgresql')
+  end
+
+  progress_message('Update the symlinks') do
+    create_links(upgrade_version)
+  end
+
+  unless progress_message('Creating temporary data directory') do
+    run_command("install -d -o gitlab-psql #{TMP_DATA_DIR}.#{upgrade_version}")
+  end
+    die 'Error creating new directory'
+  end
+
+  unless progress_message('Initializing the new database') do
+    run_pg_command(
+      "#{base_path}/embedded/bin/initdb -D #{TMP_DATA_DIR}.#{upgrade_version} " \
+      "--locale #{locale} --encoding #{encoding} --lc-collate=#{locale}.#{encoding} " \
+      "--lc-ctype=#{locale}.#{encoding}"
+    )
+  end
+    die 'Error initializing new database'
+  end
+
+  unless progress_message('Upgrading the data') do
+    run_pg_command(
+      "#{base_path}/embedded/bin/pg_upgrade -b #{base_path}/embedded/postgresql/#{default_version}/bin " \
+      "-d #{DATA_DIR} -D #{TMP_DATA_DIR}.#{upgrade_version} -B #{base_path}/embedded/bin"
+    )
+  end
+    die 'Error upgrading the database'
+  end
+
+  unless progress_message('Move the old data directory out of the way') do
+    run_command("mv #{DATA_DIR} #{TMP_DATA_DIR}.#{default_version}")
+  end
+    die 'Error moving data for older version, '
+  end
+
+  unless progress_message('Rename the new data directory') do
+    run_command("mv #{TMP_DATA_DIR}.#{upgrade_version} #{DATA_DIR}")
+  end
+    die "Error moving #{TMP_DATA_DIR}.#{upgrade_version} to #{DATA_DIR}"
+  end
+
+  log 'Upgrade is complete, doing post configuration steps'
+  unless progress_message('Running reconfigure') do
+    run_chef("#{base_path}/embedded/cookbooks/dna.json").success?
+  end
+    die 'Something went wrong during final reconfiguration, please check the output'
+  end
+  log 'Database upgrade is complete, running analyze_new_cluster.sh'
+  run_pg_command("#{DATA_DIR}/../analyze_new_cluster.sh")
+  log '==== Upgrade has completed ===='
+  log 'Please verify everything is working and run the following if so'
+  log "rm -rf #{TMP_DATA_DIR}.#{default_version}"
+  maintenance_mode('disable')
+  exit! 0
+end
+
+class ExecutionError < StandardError
+  attr_accessor :command, :stdout, :stderr
+  def initialize(command, stdout, stderr)
+    @command = command
+    @stdout = stdout
+    @stderr = stderr
+  end
+end
+
+def get_command_output(command)
+  shell_out = Mixlib::ShellOut.new(command)
+  shell_out.run_command
+  begin
+    shell_out.error!
+  rescue Mixlib::ShellOut::ShellCommandFailed
+    raise ExecutionError.new(command, shell_out.stdout, shell_out.stderr)
+  end
+  shell_out.stdout
+end
+
+def run_pg_command(command)
+  begin
+    results = get_command_output("su - gitlab-psql -c \"#{command}\"")
+  rescue ExecutionError => e
+    log "Could not run: #{command}"
+    log "STDOUT: #{e.stdout}"
+    log "STDERR: #{e.stderr}"
+    die 'Please check the output and try again'
+  end
+  results
+end
+
+def fetch_running_version
+  get_command_output("#{base_path}/embedded/bin/pg_ctl --version").split.last
+end
+
+def version_from_manifest(software)
+  if @versions.nil?
+    @versions = JSON.parse(File.read("#{base_path}/version-manifest.json"))
+  end
+  if @versions['software'].key?(software)
+    return @versions['software'][software]['described_version']
+  end
+  nil
+end
+
+def default_version
+  version_from_manifest('postgresql')
+end
+
+def upgrade_version
+  version_from_manifest('postgresql_new')
+end
+
+def fetch_lc_collate
+  run_pg_command(
+    "#{base_path}/embedded/bin/psql -h #{DATA_DIR}/.. -d postgres -c 'SHOW LC_COLLATE' -q -t"
+  )
+end
+
+def create_links(version)
+  Dir.glob("#{INST_DIR}/#{version}/bin/*").each do |bin_file|
+    destination = "#{base_path}/embedded/bin/#{File.basename(bin_file)}"
+    get_command_output("ln -sf #{bin_file} #{destination}")
+  end
+end
+
+def revert
+  log '== Reverting =='
+  run_sv_command_for_service('stop', 'postgresql')
+  if Dir.exist?("#{TMP_DATA_DIR}.#{default_version}")
+    run_command("rm -rf #{DATA_DIR}")
+    run_command("mv #{TMP_DATA_DIR}.#{default_version} #{DATA_DIR}")
+  end
+  create_links(default_version)
+  run_sv_command_for_service('start', 'postgresql')
+  log'== Reverted =='
+end
+
+def die(message)
+  log '== Fatal error =='
+  log message
+  revert
+  log "== Reverted to #{default_version}. Please check output for what went wrong =="
+  maintenance_mode('disable')
+  exit 1
+end
+
+def progress_message(message, &block)
+  $stdout.print "\r#{message}:"
+  results = block.call
+  if results
+    $stdout.print "\r#{message}: \e[32mOK\e[0m\n"
+  else
+    $stdout.print "\r#{message}: \e[31mNOT OK\e[0m\n"
+  end
+  results
+end
+
+def maintenance_mode(command)
+  # In order for the deploy page to work, we need nginx, unicorn, redis, and
+  # gitlab-workhorse running
+  # We'll manage postgresql during the ugprade process
+  omit_services = %w(postgresql nginx unicorn redis gitlab-workhorse)
+  if command.eql?('enable')
+    dp_cmd = 'up'
+    sv_cmd = 'stop'
+  elsif command.eql?('disable')
+    dp_cmd = 'down'
+    sv_cmd = 'start'
+  else
+    raise StandardError("Cannot handle command #{command}")
+  end
+  progress_message('Toggling deploy page') do
+    run_command("#{base_path}/bin/gitlab-ctl deploy-page #{dp_cmd}")
+  end
+  progress_message('Toggling services') do
+    get_all_services.select { |x| !omit_services.include?(x) }.each do |svc|
+      run_sv_command_for_service(sv_cmd, svc)
+    end
+  end
+end

spec/chef/recipes/add_trusted_certs_spec.rb

@@ -4,7 +4,9 @@ describe 'add_trusted_certs recipe' do
   let(:chef_run) { ChefSpec::SoloRunner.converge('gitlab::default') }
   let(:cert_helper) { CertificateHelper.new('/etc/gitlab/trusted-certs', '/opt/gitlab/embedded/ssl/certs', '/var/opt/gitlab') }
-  before { allow(Gitlab).to receive(:[]).and_call_original }
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
   it 'creates the certificate directories' do
     expect(chef_run).to create_directory('/opt/gitlab/embedded/ssl/certs').with(mode: '0755')

spec/chef/recipes/gitlab-rails_spec.rb

@@ -5,10 +5,6 @@ describe 'gitlab::gitlab-rails' do
   before do
     allow(Gitlab).to receive(:[]).and_call_original
-
-    # Prevent chef converge from reloading the helper library, which would override our helper stub
-    allow(Kernel).to receive(:load).and_call_original
-    allow(Kernel).to receive(:load).with(%r{gitlab/libraries/storage_directory_helper}).and_return(true)
   end
   context 'when manage-storage-directories is disabled' do

spec/chef/recipes/gitlab-shell_spec.rb

@@ -5,10 +5,6 @@ describe 'gitlab::gitlab-shell' do
   before do
     allow(Gitlab).to receive(:[]).and_call_original
-
-    # Prevent chef converge from reloading the storage helper library, which would override our helper stub
-    allow(Kernel).to receive(:load).and_call_original
-    allow(Kernel).to receive(:load).with(%r{gitlab/libraries/storage_directory_helper}).and_return(true)
   end
   it 'calls into check permissions to create and validate the authorized_keys' do
@@ -182,7 +178,9 @@ end
 describe 'gitlab_shell::git_data_dir' do
   let(:chef_run) { ChefSpec::SoloRunner.new(step_into: %w(templatesymlink)).converge('gitlab::default') }
-  before { allow(Gitlab).to receive(:[]).and_call_original }
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
   context 'when git_data_dir is set as a single directory' do
     before { stub_gitlab_rb(git_data_dir: '/tmp/user/git-data') }

spec/chef/recipes/gitlab-workhorse_spec.rb

@@ -5,10 +5,6 @@ describe 'gitlab::gitlab-workhorse' do
   before do
     allow(Gitlab).to receive(:[]).and_call_original
-
-    # Prevent chef converge from reloading the helper library, which would override our helper stub
-    allow(Kernel).to receive(:load).and_call_original
-    allow(Kernel).to receive(:load).with(%r{gitlab/libraries/storage_directory_helper}).and_return(true)
   end
   context 'with environment variables' do

spec/chef/recipes/nginx_spec.rb

@@ -73,7 +73,9 @@ describe 'nginx' do
     }
   end
-  before { allow(Gitlab).to receive(:[]).and_call_original }
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
   context 'when http external urls are being used' do
     before do

spec/chef/recipes/postgresql_spec.rb

 require 'chef_helper'
-describe 'postgresql' do
+describe 'postgresql 9.2' do
   let(:chef_run) { ChefSpec::SoloRunner.converge('gitlab::default') }
-  before { allow(Gitlab).to receive(:[]).and_call_original }
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+    allow_any_instance_of(PgHelper).to receive(:version).and_return("9.2.18")
+  end
   context 'with default settings' do
     it 'correctly sets the shared_preload_libraries default setting' do
@@ -47,4 +50,58 @@ describe 'postgresql' do
         .with_content(/log_line_prefix = '%a'/)
     end
   end
+
+  it 'sets unix_socket_directory' do
+    expect(chef_run.node['gitlab']['postgresql']['unix_socket_directory'])
+      .to eq('/var/opt/gitlab/postgresql')
+    expect(chef_run.node['gitlab']['postgresql']['unix_socket_directories'])
+      .to eq(nil)
+    expect(chef_run).to render_file(
+      '/var/opt/gitlab/postgresql/data/postgresql.conf'
+    ).with_content { |content|
+      expect(content).to match(
+        /unix_socket_directory = '\/var\/opt\/gitlab\/postgresql'/
+      )
+      expect(content).not_to match(
+        /unix_socket_directories = '\/var\/opt\/gitlab\/postgresql'/
+      )
+    }
+  end
+
+  it 'sets checkpoint_segments' do
+    expect(chef_run.node['gitlab']['postgresql']['checkpoint_segments'])
+      .to eq(10)
+    expect(chef_run).to render_file(
+      '/var/opt/gitlab/postgresql/data/postgresql.conf'
+    ).with_content(/checkpoint_segments = 10/)
+  end
+end
+
+describe 'postgresl 9.6' do
+  let(:chef_run) { ChefSpec::SoloRunner.converge('gitlab::default') }
+
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
+
+  it 'sets unix_socket_directories' do
+    expect(chef_run.node['gitlab']['postgresql']['unix_socket_directory'])
+      .to eq('/var/opt/gitlab/postgresql')
+    expect(chef_run).to render_file(
+      '/var/opt/gitlab/postgresql/data/postgresql.conf'
+    ).with_content { |content|
+      expect(content).to match(
+        /unix_socket_directories = '\/var\/opt\/gitlab\/postgresql'/
+      )
+      expect(content).not_to match(
+        /unix_socket_directory = '\/var\/opt\/gitlab\/postgresql'/
+      )
+    }
+  end
+
+  it 'does not set checkpoint_segments' do
+    expect(chef_run).not_to render_file(
+      '/var/opt/gitlab/postgresql/data/postgresql.conf'
+    ).with_content(/checkpoint_segments = 10/)
+  end
 end

spec/chef/recipes/rake-attack_spec.rb

@@ -3,7 +3,9 @@ require 'chef_helper'
 describe 'rake-attack' do
   let(:chef_run) { ChefSpec::SoloRunner.new(step_into: %w(templatesymlink)).converge('gitlab::default') }
-  before { allow(Gitlab).to receive(:[]).and_call_original }
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
   context 'when rack_attack_protected_paths is set' do
     it 'adds leading slashes' do

spec/chef/recipes/registry_spec.rb

@@ -3,7 +3,9 @@ require 'chef_helper'
 describe 'registry recipe' do
   let(:chef_run) { ChefSpec::SoloRunner.new(step_into: %w(templatesymlink)).converge('gitlab::default') }
-  before { allow(Gitlab).to receive(:[]).and_call_original }
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
   context 'when registry is enabled' do
     before { stub_gitlab_rb(registry_external_url: 'https://registry.example.com') }
@@ -116,7 +118,9 @@ end
 describe 'registry' do
   let(:chef_run) { ChefSpec::SoloRunner.new(step_into: %w(templatesymlink)).converge('gitlab::default') }
-  before { allow(Gitlab).to receive(:[]).and_call_original }
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
   context 'when registry is enabled' do
     before { stub_gitlab_rb(registry_external_url: 'https://registry.example.com') }

spec/chef_helper.rb

@@ -8,6 +8,11 @@ Dir[File.join(__dir__, '../files/gitlab-cookbooks/gitlab/libraries/*.rb')].each
 Dir[File.join(__dir__, 'support/*.rb')].each { |f| require f }
 RSpec.configure do |config|
+  def mock_file_load(file)
+    allow(Kernel).to receive(:load).and_call_original
+    allow(Kernel).to receive(:load).with(file).and_return(true)
+  end
+
   ohai_data = Ohai::System.new.tap { |ohai| ohai.all_plugins(['platform']) }.data
   platform, version = *ohai_data.values_at('platform', 'platform_version')
@@ -38,7 +43,13 @@ RSpec.configure do |config|
     stub_command("semodule -l | grep '^#gitlab-7.2.0-ssh-keygen\\s'").and_return(true)
     stub_command(%r{set \-x \&\& \[ \-d "[^"]\" \]}).and_return(false)
     stub_command(%r{set \-x \&\& \[ "\$\(stat \-\-printf='[^']*' \$\(readlink -f /[^\)]*\)\) }).and_return(false)
+    stub_command('/opt/gitlab/embedded/bin/psql --version').and_return("fake_version")
+    allow(VersionHelper).to receive(:version).and_call_original
+    allow(VersionHelper).to receive(:version).with('/opt/gitlab/embedded/bin/psql --version').and_return('fake_psql_version')
     allow_any_instance_of(Chef::Recipe).to receive(:system).with('/sbin/init --version | grep upstart')
     allow_any_instance_of(Chef::Recipe).to receive(:system).with('systemctl | grep "\-\.mount"')
+    # Prevent chef converge from reloading the storage helper library, which would override our helper stub
+    mock_file_load(%r{gitlab/libraries/storage_directory_helper})
+    mock_file_load(%r{gitlab/libraries/helper})
   end
 end