Add guidance for setting the timeout value for PG upgrades

* When an upgrade times out, advise the user on how to proceed.

changelogs/unreleased/5320-add-guidance-for-setting-the-timeout-value-for-pg-upgrades.yml

+---
+title: Add guidance for setting the timeout value for PG upgrades
+merge_request: 4563
+author:
+type: changed

files/gitlab-ctl-commands/lib/gitlab_ctl/pg_upgrade.rb

 require 'optparse'
 require 'mixlib/shellout'
+require 'rainbow'
 require_relative 'util'
 require_relative '../gitlab_ctl'
@@ -144,6 +145,14 @@ module GitlabCtl
           $stderr.puts "STDOUT: #{e.stdout}"
           $stderr.puts "STDERR: #{e.stderr}"
           false
+        rescue Mixlib::ShellOut::CommandTimeout
+          $stderr.puts
+          $stderr.puts "Timed out during the database upgrade.".color(:red)
+          $stderr.puts "To run with more time, remove the temporary directory #{tmp_data_dir}.#{target_version.major},".color(:red)
+          $stderr.puts "then re-run your previous command, adding the --timeout option.".color(:red)
+          $stderr.puts "See the docs for more information: https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server".color(:red)
+          $stderr.puts "Or run gitlab-ctl pg-upgrade --help for usage".color(:red)
+          false
         end
       end
         raise GitlabCtl::Errors::ExecutionError.new(

files/gitlab-ctl-commands/lib/gitlab_ctl/util.rb

@@ -16,12 +16,7 @@ module GitlabCtl
           raise GitlabCtl::Errors::ExecutionError.new(
             command, shell_out.stdout, shell_out.stderr
           )
-        rescue Mixlib::ShellOut::CommandTimeout
-          raise GitlabCtl::Errors::ExecutionError.new(
-            command, '', 'timed out'
-          )
         end
-
         shell_out.stdout
       end

files/gitlab-ctl-commands/pg-upgrade.rb

@@ -492,6 +492,10 @@ def analyze_cluster
     log "sudo su - #{pg_username} -c \"#{analyze_cmd}\""
     log 'If the error persists, please open an issue at: '
     log 'https://gitlab.com/gitlab-org/omnibus-gitlab/issues'
+  rescue Mixlib::ShellOut::CommandTimeout
+    $stderr.puts "Time out while running the analyze stage.".color(:yellow)
+    $stderr.puts "Please re-run the command manually as the #{pg_username} user".color(:yellow)
+    $stderr.puts analyze_command.color(:yellow)
   end
 end