Merge branch 'gitaly-disk-access-2' into 'master'

Find and mark more Git disk access locations

See merge request gitlab-org/gitlab-ce!19363

app/helpers/projects_helper.rb

@@ -412,7 +412,10 @@ module ProjectsHelper
     exports_path = File.join(Settings.shared['path'], 'tmp/project_exports')
     filtered_message = message.strip.gsub(exports_path, "[REPO EXPORT PATH]")
-    disk_path = Gitlab.config.repositories.storages[project.repository_storage].legacy_disk_path
+    disk_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+      Gitlab.config.repositories.storages[project.repository_storage].legacy_disk_path
+    end
+
     filtered_message.gsub(disk_path.chomp('/'), "[REPOS PATH]")
   end

lib/backup.rb

+module Backup
+  Error = Class.new(StandardError)
+end

lib/backup/database.rb

@@ -44,7 +44,7 @@ module Backup
       end
       report_success(success)
-      abort 'Backup failed' unless success
+      raise Backup::Error, 'Backup failed' unless success
     end
     def restore
@@ -72,7 +72,7 @@ module Backup
       end
       report_success(success)
-      abort 'Restore failed' unless success
+      abort Backup::Error, 'Restore failed' unless success
     end
     protected

lib/backup/files.rb

@@ -26,7 +26,7 @@ module Backup
         unless status.zero?
           puts output
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
         run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
@@ -39,7 +39,11 @@ module Backup
     def restore
       backup_existing_files_dir
-      run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
+      run_pipeline!([%w(gzip -cd), %W(#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
+    end
+
+    def tar
+      system(*%w[gtar --version], out: '/dev/null') ? 'gtar' : 'tar'
     end
     def backup_existing_files_dir
@@ -61,7 +65,7 @@ module Backup
     def run_pipeline!(cmd_list, options = {})
       status_list = Open3.pipeline(*cmd_list, options)
-      abort 'Backup failed' unless status_list.compact.all?(&:success?)
+      raise Backup::Error, 'Backup failed' unless status_list.compact.all?(&:success?)
     end
   end
 end

lib/backup/manager.rb

@@ -27,7 +27,7 @@ module Backup
           progress.puts "done".color(:green)
         else
           puts "creating archive #{tar_file} failed".color(:red)
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
         upload
@@ -52,7 +52,7 @@ module Backup
         progress.puts "done".color(:green)
       else
         puts "uploading backup to #{remote_directory} failed".color(:red)
-        abort 'Backup failed'
+        raise Backup::Error, 'Backup failed'
       end
     end
@@ -66,7 +66,7 @@ module Backup
           progress.puts "done".color(:green)
         else
           puts "deleting tmp directory '#{dir}' failed".color(:red)
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
       end
     end

lib/backup/repository.rb

@@ -17,7 +17,10 @@ module Backup
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{display_repo_path(project)} ... "
-        path_to_project_repo = path_to_repo(project)
+
+        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          path_to_repo(project)
+        end
         path_to_project_bundle = path_to_bundle(project)
         # Create namespace dir or hashed path if missing
@@ -51,7 +54,9 @@ module Backup
         end
         wiki = ProjectWiki.new(project)
-        path_to_wiki_repo = path_to_repo(wiki)
+        path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          path_to_repo(wiki)
+        end
         path_to_wiki_bundle = path_to_bundle(wiki)
         if File.exist?(path_to_wiki_repo)
@@ -111,7 +116,9 @@ module Backup
       # TODO: Need to find a way to do this for gitaly
       # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
       in_path(path_to_tars(project)) do |dir|
-        path_to_project_repo = path_to_repo(project)
+        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          path_to_repo(project)
+        end
         cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
         output, status = Gitlab::Popen.popen(cmd)

lib/gitlab/git/repository.rb

@@ -109,7 +109,7 @@ module Gitlab
       end
       def ==(other)
-        path == other.path
+        [storage, relative_path] == [other.storage, other.relative_path]
       end
       def path

lib/gitlab/import_export/repo_saver.rb

@@ -26,10 +26,6 @@ module Gitlab
         @shared.error(e)
         false
       end
-
-      def path_to_repo
-        @project.repository.path_to_repo
-      end
     end
   end
 end

lib/gitlab/import_export/wiki_repo_saver.rb

@@ -22,12 +22,8 @@ module Gitlab
         "project.wiki.bundle"
       end
-      def path_to_repo
-        @wiki.repository.path_to_repo
-      end
-
       def wiki_repository_exists?
-        File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty?
+        @wiki.repository.exists? && !@wiki.repository.empty?
       end
     end
   end

lib/gitlab/task_helpers.rb

@@ -128,10 +128,12 @@ module Gitlab
     end
     def all_repos
-      Gitlab.config.repositories.storages.each_value do |repository_storage|
-        IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find|
-          find.each_line do |path|
-            yield path.chomp
+      Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        Gitlab.config.repositories.storages.each_value do |repository_storage|
+          IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find|
+            find.each_line do |path|
+              yield path.chomp
+            end
           end
         end
       end

lib/tasks/gitlab/cleanup.rake

@@ -12,7 +12,7 @@ namespace :gitlab do
       namespaces  = Namespace.pluck(:path)
       namespaces << HASHED_REPOSITORY_NAME  # add so that it will be ignored
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        git_base_path = repository_storage.legacy_disk_path
+        git_base_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path }
         all_dirs = Dir.glob(git_base_path + '/*')
         puts git_base_path.color(:yellow)
@@ -54,7 +54,8 @@ namespace :gitlab do
       move_suffix = "+orphaned+#{Time.now.to_i}"
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_root = repository_storage.legacy_disk_path
+        repo_root = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path }
+
         # Look for global repos (legacy, depth 1) and normal repos (depth 2)
         IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
           find.each_line do |path|

spec/features/projects/import_export/import_file_spec.rb

@@ -87,11 +87,13 @@ feature 'Import/Export - project import integration test', :js do
   def wiki_exists?(project)
     wiki = ProjectWiki.new(project)
-    File.exist?(wiki.repository.path_to_repo) && !wiki.repository.empty?
+    wiki.repository.exists? && !wiki.repository.empty?
   end
   def project_hook_exists?(project)
-    Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
+    Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+      Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
+    end
   end
   def click_import_project_tab

spec/lib/backup/files_spec.rb

@@ -46,7 +46,9 @@ describe Backup::Files do
       end
       it 'calls tar command with unlink' do
-        expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
+        expect(subject).to receive(:tar).and_return('blabla-tar')
+
+        expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(blabla-tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
         subject.restore
       end
     end

spec/lib/backup/repository_spec.rb

@@ -34,7 +34,9 @@ describe Backup::Repository do
     let(:timestamp) { Time.utc(2017, 3, 22) }
     let(:temp_dirs) do
       Gitlab.config.repositories.storages.map do |name, storage|
-        File.join(storage.legacy_disk_path, '..', 'repositories.old.' + timestamp.to_i.to_s)
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          File.join(storage.legacy_disk_path, '..', 'repositories.old.' + timestamp.to_i.to_s)
+        end
       end
     end

spec/lib/gitlab/bare_repository_import/importer_spec.rb

@@ -20,6 +20,13 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
     Rainbow.enabled = @rainbow
   end
+  around do |example|
+    # TODO migrate BareRepositoryImport https://gitlab.com/gitlab-org/gitaly/issues/953
+    Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+      example.run
+    end
+  end
+
   shared_examples 'importing a repository' do
     describe '.execute' do
       it 'creates a project for a repository in storage' do

spec/lib/gitlab/bare_repository_import/repository_spec.rb

@@ -62,8 +62,10 @@ describe ::Gitlab::BareRepositoryImport::Repository do
     before do
       gitlab_shell.create_repository(repository_storage, hashed_path)
-      repository = Rugged::Repository.new(repo_path)
-      repository.config['gitlab.fullpath'] = 'to/repo'
+      Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        repository = Rugged::Repository.new(repo_path)
+        repository.config['gitlab.fullpath'] = 'to/repo'
+      end
     end
     after do

spec/lib/gitlab/git/repository_spec.rb

@@ -159,6 +159,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
     let(:feature2) { 'feature2' }
     around do |example|
+      # discover_default_branch will be moved to gitaly-ruby
       Gitlab::GitalyClient::StorageSettings.allow_disk_access do
         example.run
       end
@@ -373,6 +374,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
   context '#submodules' do
     around do |example|
+      # TODO #submodules will be removed, has been migrated to gitaly
       Gitlab::GitalyClient::StorageSettings.allow_disk_access do
         example.run
       end
@@ -1055,6 +1057,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
   describe "#rugged_commits_between" do
     around do |example|
+      # TODO #rugged_commits_between will be removed, has been migrated to gitaly
       Gitlab::GitalyClient::StorageSettings.allow_disk_access do
         example.run
       end
@@ -1703,6 +1706,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
     let(:refs) { ['deadbeef', SeedRepo::RubyBlob::ID, '909e6157199'] }
     around do |example|
+      # TODO #batch_existence isn't used anywhere, can we remove it?
       Gitlab::GitalyClient::StorageSettings.allow_disk_access do
         example.run
       end

spec/lib/gitlab/git/rev_list_spec.rb

@@ -9,9 +9,11 @@ describe Gitlab::Git::RevList do
   end
   def stub_popen_rev_list(*additional_args, with_lazy_block: true, output:)
+    repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository.path }
+
     params = [
       args_for_popen(additional_args),
-      repository.path,
+      repo_path,
       {},
       hash_including(lazy_block: with_lazy_block ? anything : nil)
     ]

spec/lib/gitlab/git_access_spec.rb

@@ -552,7 +552,7 @@ describe Gitlab::GitAccess do
       it 'returns not found' do
         project.add_guest(user)
         repo = project.repository
-        FileUtils.rm_rf(repo.path)
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access { FileUtils.rm_rf(repo.path) }
         # Sanity check for rm_rf
         expect(repo.exists?).to eq(false)
@@ -750,20 +750,22 @@ describe Gitlab::GitAccess do
     def merge_into_protected_branch
       @protected_branch_merge_commit ||= begin
-        stub_git_hooks
-        project.repository.add_branch(user, unprotected_branch, 'feature')
-        target_branch = project.repository.lookup('feature')
-        source_branch = project.repository.create_file(
-          user,
-          'filename',
-          'This is the file content',
-          message: 'This is a good commit message',
-          branch_name: unprotected_branch)
-        rugged = project.repository.rugged
-        author = { email: "email@example.com", time: Time.now, name: "Example Git User" }
-
-        merge_index = rugged.merge_commits(target_branch, source_branch)
-        Rugged::Commit.create(rugged, author: author, committer: author, message: "commit message", parents: [target_branch, source_branch], tree: merge_index.write_tree(rugged))
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          stub_git_hooks
+          project.repository.add_branch(user, unprotected_branch, 'feature')
+          target_branch = project.repository.lookup('feature')
+          source_branch = project.repository.create_file(
+            user,
+            'filename',
+            'This is the file content',
+            message: 'This is a good commit message',
+            branch_name: unprotected_branch)
+          rugged = project.repository.rugged
+          author = { email: "email@example.com", time: Time.now, name: "Example Git User" }
+
+          merge_index = rugged.merge_commits(target_branch, source_branch)
+          Rugged::Commit.create(rugged, author: author, committer: author, message: "commit message", parents: [target_branch, source_branch], tree: merge_index.write_tree(rugged))
+        end
       end
     end

spec/lib/gitlab/import_export/fork_spec.rb

@@ -41,8 +41,10 @@ describe 'forked project import' do
   after do
     FileUtils.rm_rf(export_path)
-    FileUtils.rm_rf(project_with_repo.repository.path_to_repo)
-    FileUtils.rm_rf(project.repository.path_to_repo)
+    Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+      FileUtils.rm_rf(project_with_repo.repository.path_to_repo)
+      FileUtils.rm_rf(project.repository.path_to_repo)
+    end
   end
   it 'can access the MR' do