Make backups more space-efficient

lib/backup/builds.rb

 module Backup
   class Builds
-    attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
+    attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
     def initialize
       @app_builds_dir = File.realpath(Rails.root.join('builds'))
       @backup_dir = GitlabCi.config.backup.path
-      @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'builds')
+      @backup_builds_tarball = File.join(GitlabCi.config.backup.path, 'builds/builds.tar.gz')
     end
     # Copy builds from builds directory to backup/builds
     def dump
-      FileUtils.mkdir_p(backup_builds_dir)
-      FileUtils.cp_r(app_builds_dir, backup_dir)
+      FileUtils.mkdir_p(File.dirname(backup_builds_tarball))
+      FileUtils.rm_f(backup_builds_tarball)
+
+      system(
+        *%W(tar -C #{app_builds_dir} -czf - -- .),
+        out: [backup_builds_tarball, 'w', 0600]
+      )
     end
     def restore
       backup_existing_builds_dir
-      FileUtils.cp_r(backup_builds_dir, app_builds_dir)
+      system(
+        *%W(tar -C #{app_builds_dir} -xzf - -- .),
+        in: backup_builds_tarball
+      )
     end
     def backup_existing_builds_dir

lib/backup/database.rb

@@ -11,24 +11,37 @@ module Backup
     end
     def dump
-      success = case config["adapter"]
+      FileUtils.rm_f(db_file_name)
+      compress_rd, compress_wr = IO.pipe
+      compress_pid = spawn(*%W(gzip -c), in: compress_rd, out: [db_file_name, 'w', 0600])
+      compress_rd.close
+
+      dump_pid = case config["adapter"]
       when /^mysql/ then
         $progress.print "Dumping MySQL database #{config['database']} ... "
-        system('mysqldump', *mysql_args, config['database'], out: db_file_name)
+        spawn('mysqldump', *mysql_args, config['database'], out: compress_wr)
       when "postgresql" then
         $progress.print "Dumping PostgreSQL database #{config['database']} ... "
         pg_env
-        system('pg_dump', config['database'], out: db_file_name)
+        spawn('pg_dump', config['database'], out: compress_wr)
       end
+      compress_wr.close
+
+      success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+
       report_success(success)
       abort 'Backup failed' unless success
     end
     def restore
-      success = case config["adapter"]
+      decompress_rd, decompress_wr = IO.pipe
+      decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+      decompress_wr.close
+
+      restore_pid = case config["adapter"]
       when /^mysql/ then
         $progress.print "Restoring MySQL database #{config['database']} ... "
-        system('mysql', *mysql_args, config['database'], in: db_file_name)
+        spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
       when "postgresql" then
         $progress.print "Restoring PostgreSQL database #{config['database']} ... "
         # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE
@@ -36,8 +49,12 @@ module Backup
         drop_all_tables
         drop_all_postgres_sequences
         pg_env
-        system('psql', config['database'], '-f', db_file_name)
+        spawn('psql', config['database'], in: decompress_rd)
       end
+      decompress_rd.close
+
+      success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+
       report_success(success)
       abort 'Restore failed' unless success
     end
@@ -45,7 +62,7 @@ module Backup
     protected
     def db_file_name
-      File.join(db_dir, 'database.sql')
+      File.join(db_dir, 'database.sql.gz')
     end
     def mysql_args

lib/backup/manager.rb

@@ -7,7 +7,7 @@ module Backup
       s[:backup_created_at]  = Time.now
       s[:gitlab_version]     = GitlabCi::VERSION
       s[:tar_version]        = tar_version
-      tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz"
+      tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar"
       Dir.chdir(GitlabCi.config.backup.path) do
         File.open("#{GitlabCi.config.backup.path}/backup_information.yml",
@@ -20,7 +20,7 @@ module Backup
         # create archive
         $progress.print "Creating backup archive: #{tar_file} ... "
         orig_umask = File.umask(0077)
-        if Kernel.system('tar', '-czf', tar_file, *backup_contents)
+        if Kernel.system('tar', '-cf', tar_file, *backup_contents)
           $progress.puts "done".green
         else
           puts "creating archive #{tar_file} failed".red
@@ -78,11 +78,11 @@ module Backup
         removed = 0
         
         Dir.chdir(GitlabCi.config.backup.path) do
-          file_list = Dir.glob('*_gitlab_ci_backup.tar.gz')
-          file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ }
+          file_list = Dir.glob('*_gitlab_ci_backup.tar')
+          file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar/ }
           file_list.sort.each do |timestamp|
             if Time.at(timestamp) < (Time.now - keep_time)
-              if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz))
+              if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar))
                 removed += 1
               end
             end
@@ -99,7 +99,7 @@ module Backup
       Dir.chdir(GitlabCi.config.backup.path)
       # check for existing backups in the backup dir
-      file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i }
+      file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
       puts "no backups found" if file_list.count == 0
       if file_list.count > 1 && ENV["BACKUP"].nil?
@@ -108,7 +108,7 @@ module Backup
         exit 1
       end
-      tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz")
+      tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
       unless File.exists?(tar_file)
         puts "The specified backup doesn't exist!"
@@ -117,7 +117,7 @@ module Backup
       $progress.print "Unpacking backup ... "
-      unless Kernel.system(*%W(tar -xzf #{tar_file}))
+      unless Kernel.system(*%W(tar -xf #{tar_file}))
         puts "unpacking backup failed".red
         exit 1
       else