Merge branch 'backup-cron-mode' into 'master'

Backup cron mode

If you have your server configured to receive emails containing the output of
cron jobs it is annoying to get long spammy emails from the backup script. This
change adds a 'cron mode' that makes the backup script silent unless something
goes wrong during the backup.

See merge request !1268

CHANGELOG

 v 7.6.0
   - Fork repository to groups
   - New rugged version
-  -
+  - Add CRON=1 backup setting for quiet backups
   -
   -
   -

doc/raketasks/backup_restore.md

@@ -203,5 +203,8 @@ Add the following lines at the bottom:
 ```
 # Create a full backup of the GitLab repositories and SQL database every day at 4am
-0 4 * * * cd /home/git/gitlab && PATH=/usr/local/bin:/usr/bin:/bin bundle exec rake gitlab:backup:create RAILS_ENV=production
+0 4 * * * cd /home/git/gitlab && PATH=/usr/local/bin:/usr/bin:/bin bundle exec rake gitlab:backup:create RAILS_ENV=production CRON=1
 ```
+
+The `CRON=1` environment setting tells the backup script to suppress all progress output if there are no errors.
+This is recommended to reduce cron spam.

lib/backup/database.rb

@@ -13,10 +13,10 @@ module Backup
     def dump
       success = case config["adapter"]
       when /^mysql/ then
-        print "Dumping MySQL database #{config['database']} ... "
+        $progress.print "Dumping MySQL database #{config['database']} ... "
         system('mysqldump', *mysql_args, config['database'], out: db_file_name)
       when "postgresql" then
-        print "Dumping PostgreSQL database #{config['database']} ... "
+        $progress.print "Dumping PostgreSQL database #{config['database']} ... "
         pg_env
         system('pg_dump', config['database'], out: db_file_name)
       end
@@ -27,10 +27,10 @@ module Backup
     def restore
       success = case config["adapter"]
       when /^mysql/ then
-        print "Restoring MySQL database #{config['database']} ... "
+        $progress.print "Restoring MySQL database #{config['database']} ... "
         system('mysql', *mysql_args, config['database'], in: db_file_name)
       when "postgresql" then
-        print "Restoring PostgreSQL database #{config['database']} ... "
+        $progress.print "Restoring PostgreSQL database #{config['database']} ... "
         # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE
         # statements like MySQL.
         Rake::Task["gitlab:db:drop_all_tables"].invoke
@@ -69,9 +69,9 @@ module Backup
     def report_success(success)
       if success
-        puts '[DONE]'.green
+        $progress.puts '[DONE]'.green
       else
-        puts '[FAILED]'.red
+        $progress.puts '[FAILED]'.red
       end
     end
   end

lib/backup/manager.rb

@@ -18,11 +18,11 @@ module Backup
       end
       # create archive
-      print "Creating backup archive: #{tar_file} ... "
+      $progress.print "Creating backup archive: #{tar_file} ... "
       if Kernel.system('tar', '-cf', tar_file, *BACKUP_CONTENTS)
-        puts "done".green
+        $progress.puts "done".green
       else
-        puts "failed".red
+        puts "creating archive #{tar_file} failed".red
         abort 'Backup failed'
       end
@@ -31,37 +31,37 @@ module Backup
     def upload(tar_file)
       remote_directory = Gitlab.config.backup.upload.remote_directory
-      print "Uploading backup archive to remote storage #{remote_directory} ... "
+      $progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
       connection_settings = Gitlab.config.backup.upload.connection
       if connection_settings.blank?
-        puts "skipped".yellow
+        $progress.puts "skipped".yellow
         return
       end
       connection = ::Fog::Storage.new(connection_settings)
       directory = connection.directories.get(remote_directory)
       if directory.files.create(key: tar_file, body: File.open(tar_file), public: false)
-        puts "done".green
+        $progress.puts "done".green
       else
-        puts "failed".red
+        puts "uploading backup to #{remote_directory} failed".red
         abort 'Backup failed'
       end
     end
     def cleanup
-      print "Deleting tmp directories ... "
+      $progress.print "Deleting tmp directories ... "
       if Kernel.system('rm', '-rf', *BACKUP_CONTENTS)
-        puts "done".green
+        $progress.puts "done".green
       else
-        puts "failed".red
+        puts "deleting tmp directory failed".red
         abort 'Backup failed'
       end
     end
     def remove_old
       # delete backups
-      print "Deleting old backups ... "
+      $progress.print "Deleting old backups ... "
       keep_time = Gitlab.config.backup.keep_time.to_i
       path = Gitlab.config.backup.path
@@ -76,9 +76,9 @@ module Backup
             end
           end
         end
-        puts "done. (#{removed} removed)".green
+        $progress.puts "done. (#{removed} removed)".green
       else
-        puts "skipping".yellow
+        $progress.puts "skipping".yellow
       end
     end
@@ -101,12 +101,12 @@ module Backup
         exit 1
       end
-      print "Unpacking backup ... "
+      $progress.print "Unpacking backup ... "
       unless Kernel.system(*%W(tar -xf #{tar_file}))
-        puts "failed".red
+        puts "unpacking backup failed".red
         exit 1
       else
-        puts "done".green
+        $progress.puts "done".green
       end
       settings = YAML.load_file("backup_information.yml")

lib/backup/repository.rb

@@ -8,19 +8,21 @@ module Backup
       prepare
       Project.find_each(batch_size: 1000) do |project|
-        print " * #{project.path_with_namespace} ... "
+        $progress.print " * #{project.path_with_namespace} ... "
         # Create namespace dir if missing
         FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace
         if project.empty_repo?
-          puts "[SKIPPED]".cyan
+          $progress.puts "[SKIPPED]".cyan
         else
-          output, status = Gitlab::Popen.popen(%W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all))
+          cmd = %W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all)
+          output, status = Gitlab::Popen.popen(cmd)
           if status.zero?
-            puts "[DONE]".green
+            $progress.puts "[DONE]".green
           else
             puts "[FAILED]".red
+            puts "failed: #{cmd.join(' ')}"
             puts output
             abort 'Backup failed'
           end
@@ -29,15 +31,17 @@ module Backup
         wiki = ProjectWiki.new(project)
         if File.exists?(path_to_repo(wiki))
-          print " * #{wiki.path_with_namespace} ... "
+          $progress.print " * #{wiki.path_with_namespace} ... "
           if wiki.repository.empty?
-            puts " [SKIPPED]".cyan
+            $progress.puts " [SKIPPED]".cyan
           else
-            output, status = Gitlab::Popen.popen(%W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all))
+            cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
+            output, status = Gitlab::Popen.popen(cmd)
             if status.zero?
-              puts " [DONE]".green
+              $progress.puts " [DONE]".green
             else
               puts " [FAILED]".red
+              puts "failed: #{cmd.join(' ')}"
               abort 'Backup failed'
             end
           end
@@ -55,7 +59,7 @@ module Backup
       FileUtils.mkdir_p(repos_path)
       Project.find_each(batch_size: 1000) do |project|
-        print "#{project.path_with_namespace} ... "
+        $progress.print "#{project.path_with_namespace} ... "
         project.namespace.ensure_dir_exist if project.namespace
@@ -66,30 +70,35 @@ module Backup
         end
         if system(*cmd, silent)
-          puts "[DONE]".green
+          $progress.puts "[DONE]".green
         else
           puts "[FAILED]".red
+          puts "failed: #{cmd.join(' ')}"
           abort 'Restore failed'
         end
         wiki = ProjectWiki.new(project)
         if File.exists?(path_to_bundle(wiki))
-          print " * #{wiki.path_with_namespace} ... "
-          if system(*%W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}), silent)
-            puts " [DONE]".green
+          $progress.print " * #{wiki.path_with_namespace} ... "
+          cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
+          if system(*cmd, silent)
+            $progress.puts " [DONE]".green
           else
             puts " [FAILED]".red
+            puts "failed: #{cmd.join(' ')}"
             abort 'Restore failed'
           end
         end
       end
-      print 'Put GitLab hooks in repositories dirs'.yellow
-      if system("#{Gitlab.config.gitlab_shell.path}/bin/create-hooks")
-        puts " [DONE]".green
+      $progress.print 'Put GitLab hooks in repositories dirs'.yellow
+      cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks"
+      if system(cmd)
+        $progress.puts " [DONE]".green
       else
         puts " [FAILED]".red
+        puts "failed: #{cmd}"
       end
     end

lib/tasks/gitlab/backup.rake

@@ -6,6 +6,7 @@ namespace :gitlab do
     desc "GITLAB | Create a backup of the GitLab system"
     task create: :environment do
       warn_user_is_not_gitlab
+      configure_cron_mode
       Rake::Task["gitlab:backup:db:create"].invoke
       Rake::Task["gitlab:backup:repo:create"].invoke
@@ -21,6 +22,7 @@ namespace :gitlab do
     desc "GITLAB | Restore a previously created backup"
     task restore: :environment do
       warn_user_is_not_gitlab
+      configure_cron_mode
       backup = Backup::Manager.new
       backup.unpack
@@ -35,43 +37,54 @@ namespace :gitlab do
     namespace :repo do
       task create: :environment do
-        puts "Dumping repositories ...".blue
+        $progress.puts "Dumping repositories ...".blue
         Backup::Repository.new.dump
-        puts "done".green
+        $progress.puts "done".green
       end
       task restore: :environment do
-        puts "Restoring repositories ...".blue
+        $progress.puts "Restoring repositories ...".blue
         Backup::Repository.new.restore
-        puts "done".green
+        $progress.puts "done".green
       end
     end
     namespace :db do
       task create: :environment do
-        puts "Dumping database ... ".blue
+        $progress.puts "Dumping database ... ".blue
         Backup::Database.new.dump
-        puts "done".green
+        $progress.puts "done".green
       end
       task restore: :environment do
-        puts "Restoring database ... ".blue
+        $progress.puts "Restoring database ... ".blue
         Backup::Database.new.restore
-        puts "done".green
+        $progress.puts "done".green
       end
     end
     namespace :uploads do
       task create: :environment do
-        puts "Dumping uploads ... ".blue
+        $progress.puts "Dumping uploads ... ".blue
         Backup::Uploads.new.dump
-        puts "done".green
+        $progress.puts "done".green
       end
       task restore: :environment do
-        puts "Restoring uploads ... ".blue
+        $progress.puts "Restoring uploads ... ".blue
         Backup::Uploads.new.restore
-        puts "done".green
+        $progress.puts "done".green
+      end
+    end
+
+    def configure_cron_mode
+      if ENV['CRON']
+        # We need an object we can say 'puts' and 'print' to; let's use a
+        # StringIO.
+        require 'stringio'
+        $progress = StringIO.new
+      else
+        $progress = $stdout
       end
     end
   end # namespace end: backup