Merge branch 'incremental_repository_backup' into 'master'

Create feature flag for incremental repository backups

See merge request gitlab-org/gitlab!79589

config/feature_flags/development/incremental_repository_backup.yml

+---
+name: incremental_repository_backup
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79589
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/355945
+milestone: '14.9'
+type: development
+group: group::gitaly
+default_enabled: false

doc/raketasks/backup_restore.md

@@ -1854,3 +1854,22 @@ To enable it:
 ```ruby
 Feature.enable(:gitaly_backup)
 ```
+
+### Incremental repository backups
+
+> Introduced in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `incremental_repository_backup`. Disabled by default.
+
+FLAG:
+On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `incremental_repository_backup`.
+On GitLab.com, this feature is not available.
+This feature is not ready for production use.
+
+Incremental backups can be faster than full backups because they only pack changes since the last backup into the backup
+bundle for each repository. Because incremental backups require access to the previous backup, you can't use incremental
+backups with tar files.
+
+To create an incremental backup, run:
+
+```shell
+sudo gitlab-backup create SKIP=tar INCREMENTAL=yes
+```

lib/backup/gitaly_backup.rb

@@ -9,10 +9,13 @@ module Backup
     # @param [StringIO] progress IO interface to output progress
     # @param [Integer] max_parallelism max parallelism when running backups
     # @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism)
-    def initialize(progress, max_parallelism: nil, storage_parallelism: nil)
+    # @param [String] backup_id unique identifier for the backup
+    def initialize(progress, max_parallelism: nil, storage_parallelism: nil, incremental: false, backup_id: nil)
       @progress = progress
       @max_parallelism = max_parallelism
       @storage_parallelism = storage_parallelism
+      @incremental = incremental
+      @backup_id = backup_id
     end
     def start(type, backup_repos_path)
@@ -30,6 +33,13 @@ module Backup
       args = []
       args += ['-parallel', @max_parallelism.to_s] if @max_parallelism
       args += ['-parallel-storage', @storage_parallelism.to_s] if @storage_parallelism
+      if Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml)
+        args += ['-layout', 'pointer']
+        if type == :create
+          args += ['-incremental'] if @incremental
+          args += ['-id', @backup_id] if @backup_id
+        end
+      end
       @input_stream, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args)

lib/backup/manager.rb

@@ -21,6 +21,7 @@ module Backup
       max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i
       max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i
       force = ENV['force'] == 'yes'
+      incremental = Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false)
       @definitions = definitions || {
         'db' => TaskDefinition.new(
@@ -32,7 +33,7 @@ module Backup
           destination_path: 'repositories',
           destination_optional: true,
           task: Repositories.new(progress,
-                                 strategy: repository_backup_strategy,
+                                 strategy: repository_backup_strategy(incremental),
                                  max_concurrency: max_concurrency,
                                  max_storage_concurrency: max_storage_concurrency)
         ),
@@ -481,11 +482,11 @@ module Backup
       Gitlab.config.backup.upload.connection&.provider&.downcase == 'google'
     end
-    def repository_backup_strategy
+    def repository_backup_strategy(incremental)
       if Feature.enabled?(:gitaly_backup, default_enabled: :yaml)
         max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
         max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
-        Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
+        Backup::GitalyBackup.new(progress, incremental: incremental, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
       else
         Backup::GitalyRpcBackup.new(progress)
       end

spec/lib/backup/gitaly_backup_spec.rb

@@ -6,6 +6,7 @@ RSpec.describe Backup::GitalyBackup do
   let(:max_parallelism) { nil }
   let(:storage_parallelism) { nil }
   let(:destination) { File.join(Gitlab.config.backup.path, 'repositories') }
+  let(:backup_id) { '20220101' }
   let(:progress) do
     Tempfile.new('progress').tap do |progress|
@@ -24,7 +25,7 @@ RSpec.describe Backup::GitalyBackup do
     progress.close
   end
-  subject { described_class.new(progress, max_parallelism: max_parallelism, storage_parallelism: storage_parallelism) }
+  subject { described_class.new(progress, max_parallelism: max_parallelism, storage_parallelism: storage_parallelism, backup_id: backup_id) }
   context 'unknown' do
     it 'fails to start unknown' do
@@ -41,7 +42,7 @@ RSpec.describe Backup::GitalyBackup do
         project_snippet = create(:project_snippet, :repository, project: project)
         personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
-        expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything).and_call_original
+        expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-layout', 'pointer', '-id', backup_id).and_call_original
         subject.start(:create, destination)
         subject.enqueue(project, Gitlab::GlRepository::PROJECT)
@@ -51,18 +52,18 @@ RSpec.describe Backup::GitalyBackup do
         subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
         subject.finish!
-        expect(File).to exist(File.join(destination, project.disk_path + '.bundle'))
-        expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle'))
-        expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle'))
-        expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle'))
-        expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle'))
+        expect(File).to exist(File.join(destination, project.disk_path, backup_id, '001.bundle'))
+        expect(File).to exist(File.join(destination, project.disk_path + '.wiki', backup_id, '001.bundle'))
+        expect(File).to exist(File.join(destination, project.disk_path + '.design', backup_id, '001.bundle'))
+        expect(File).to exist(File.join(destination, personal_snippet.disk_path, backup_id, '001.bundle'))
+        expect(File).to exist(File.join(destination, project_snippet.disk_path, backup_id, '001.bundle'))
       end
       context 'parallel option set' do
         let(:max_parallelism) { 3 }
         it 'passes parallel option through' do
-          expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel', '3').and_call_original
+          expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel', '3', '-layout', 'pointer', '-id', backup_id).and_call_original
           subject.start(:create, destination)
           subject.finish!
@@ -73,7 +74,7 @@ RSpec.describe Backup::GitalyBackup do
         let(:storage_parallelism) { 3 }
         it 'passes parallel option through' do
-          expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel-storage', '3').and_call_original
+          expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel-storage', '3', '-layout', 'pointer', '-id', backup_id).and_call_original
           subject.start(:create, destination)
           subject.finish!
@@ -86,6 +87,36 @@ RSpec.describe Backup::GitalyBackup do
         subject.start(:create, destination)
         expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
       end
+
+      context 'feature flag incremental_repository_backup disabled' do
+        before do
+          stub_feature_flags(incremental_repository_backup: false)
+        end
+
+        it 'creates repository bundles', :aggregate_failures do
+          # Add data to the wiki, design repositories, and snippets, so they will be included in the dump.
+          create(:wiki_page, container: project)
+          create(:design, :with_file, issue: create(:issue, project: project))
+          project_snippet = create(:project_snippet, :repository, project: project)
+          personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
+
+          expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything).and_call_original
+
+          subject.start(:create, destination)
+          subject.enqueue(project, Gitlab::GlRepository::PROJECT)
+          subject.enqueue(project, Gitlab::GlRepository::WIKI)
+          subject.enqueue(project, Gitlab::GlRepository::DESIGN)
+          subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
+          subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
+          subject.finish!
+
+          expect(File).to exist(File.join(destination, project.disk_path + '.bundle'))
+          expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle'))
+          expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle'))
+          expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle'))
+          expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle'))
+        end
+      end
     end
     context 'hashed storage' do
@@ -113,7 +144,7 @@ RSpec.describe Backup::GitalyBackup do
       end
       it 'passes through SSL envs' do
-        expect(Open3).to receive(:popen2).with(ssl_env, anything, 'create', '-path', anything).and_call_original
+        expect(Open3).to receive(:popen2).with(ssl_env, anything, 'create', '-path', anything, '-layout', 'pointer', '-id', backup_id).and_call_original
         subject.start(:create, destination)
         subject.finish!
@@ -138,7 +169,7 @@ RSpec.describe Backup::GitalyBackup do
       copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle')
       copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle')
-      expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything).and_call_original
+      expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-layout', 'pointer').and_call_original
       subject.start(:restore, destination)
       subject.enqueue(project, Gitlab::GlRepository::PROJECT)
@@ -150,18 +181,18 @@ RSpec.describe Backup::GitalyBackup do
       collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
-      expect(collect_commit_shas.call(project.repository)).to eq(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec'])
-      expect(collect_commit_shas.call(project.wiki.repository)).to eq(['c74b9948d0088d703ee1fafeddd9ed9add2901ea'])
-      expect(collect_commit_shas.call(project.design_repository)).to eq(['c3cd4d7bd73a51a0f22045c3a4c871c435dc959d'])
-      expect(collect_commit_shas.call(personal_snippet.repository)).to eq(['3b3c067a3bc1d1b695b51e2be30c0f8cf698a06e'])
-      expect(collect_commit_shas.call(project_snippet.repository)).to eq(['6e44ba56a4748be361a841e759c20e421a1651a1'])
+      expect(collect_commit_shas.call(project.repository)).to match_array(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec'])
+      expect(collect_commit_shas.call(project.wiki.repository)).to match_array(['c74b9948d0088d703ee1fafeddd9ed9add2901ea'])
+      expect(collect_commit_shas.call(project.design_repository)).to match_array(['c3cd4d7bd73a51a0f22045c3a4c871c435dc959d'])
+      expect(collect_commit_shas.call(personal_snippet.repository)).to match_array(['3b3c067a3bc1d1b695b51e2be30c0f8cf698a06e'])
+      expect(collect_commit_shas.call(project_snippet.repository)).to match_array(['6e44ba56a4748be361a841e759c20e421a1651a1'])
     end
     context 'parallel option set' do
       let(:max_parallelism) { 3 }
       it 'passes parallel option through' do
-        expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel', '3').and_call_original
+        expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel', '3', '-layout', 'pointer').and_call_original
         subject.start(:restore, destination)
         subject.finish!
@@ -172,13 +203,45 @@ RSpec.describe Backup::GitalyBackup do
       let(:storage_parallelism) { 3 }
       it 'passes parallel option through' do
-        expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel-storage', '3').and_call_original
+        expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel-storage', '3', '-layout', 'pointer').and_call_original
         subject.start(:restore, destination)
         subject.finish!
       end
     end
+    context 'feature flag incremental_repository_backup disabled' do
+      before do
+        stub_feature_flags(incremental_repository_backup: false)
+      end
+
+      it 'restores from repository bundles', :aggregate_failures do
+        copy_bundle_to_backup_path('project_repo.bundle', project.disk_path + '.bundle')
+        copy_bundle_to_backup_path('wiki_repo.bundle', project.disk_path + '.wiki.bundle')
+        copy_bundle_to_backup_path('design_repo.bundle', project.disk_path + '.design.bundle')
+        copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle')
+        copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle')
+
+        expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything).and_call_original
+
+        subject.start(:restore, destination)
+        subject.enqueue(project, Gitlab::GlRepository::PROJECT)
+        subject.enqueue(project, Gitlab::GlRepository::WIKI)
+        subject.enqueue(project, Gitlab::GlRepository::DESIGN)
+        subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
+        subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
+        subject.finish!
+
+        collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
+
+        expect(collect_commit_shas.call(project.repository)).to match_array(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec'])
+        expect(collect_commit_shas.call(project.wiki.repository)).to match_array(['c74b9948d0088d703ee1fafeddd9ed9add2901ea'])
+        expect(collect_commit_shas.call(project.design_repository)).to match_array(['c3cd4d7bd73a51a0f22045c3a4c871c435dc959d'])
+        expect(collect_commit_shas.call(personal_snippet.repository)).to match_array(['3b3c067a3bc1d1b695b51e2be30c0f8cf698a06e'])
+        expect(collect_commit_shas.call(project_snippet.repository)).to match_array(['6e44ba56a4748be361a841e759c20e421a1651a1'])
+      end
+    end
+
     it 'raises when the exit code not zero' do
       expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))

spec/tasks/gitlab/backup_rake_spec.rb

@@ -176,8 +176,8 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           expect(exit_status).to eq(0)
           expect(tar_contents).to match(user_backup_path)
-          expect(tar_contents).to match("#{user_backup_path}/custom_hooks.tar")
-          expect(tar_contents).to match("#{user_backup_path}.bundle")
+          expect(tar_contents).to match("#{user_backup_path}/.+/001.custom_hooks.tar")
+          expect(tar_contents).to match("#{user_backup_path}/.+/001.bundle")
         end
         it 'restores files correctly' do
@@ -360,14 +360,14 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           expect(exit_status).to eq(0)
           [
-            "#{project_a.disk_path}.bundle",
-            "#{project_a.disk_path}.wiki.bundle",
-            "#{project_a.disk_path}.design.bundle",
-            "#{project_b.disk_path}.bundle",
-            "#{project_snippet_a.disk_path}.bundle",
-            "#{project_snippet_b.disk_path}.bundle"
+            "#{project_a.disk_path}/.+/001.bundle",
+            "#{project_a.disk_path}.wiki/.+/001.bundle",
+            "#{project_a.disk_path}.design/.+/001.bundle",
+            "#{project_b.disk_path}/.+/001.bundle",
+            "#{project_snippet_a.disk_path}/.+/001.bundle",
+            "#{project_snippet_b.disk_path}/.+/001.bundle"
           ].each do |repo_name|
-            expect(tar_lines.grep(/#{repo_name}/).size).to eq 1
+            expect(tar_lines).to include(a_string_matching(repo_name))
           end
         end
@@ -428,7 +428,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect(::Backup::Repositories).to receive(:new)
           .with(anything, strategy: anything, max_concurrency: 5, max_storage_concurrency: 2)
           .and_call_original
-        expect(::Backup::GitalyBackup).to receive(:new).with(anything, max_parallelism: 5, storage_parallelism: 2).and_call_original
+        expect(::Backup::GitalyBackup).to receive(:new).with(anything, max_parallelism: 5, storage_parallelism: 2, incremental: false).and_call_original
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
       end