Skip to content
Snippets Groups Projects
Commit a4e75e7a authored by Jacob Vosmaer (GitLab)'s avatar Jacob Vosmaer (GitLab)
Browse files

Use Gitaly for fetches and creating bundles

parent 5e471c78
No related branches found
No related tags found
1 merge request!10495Merge Requests - Assignee
Hide whitespace changes
Inline Side-by-side
lib/gitlab/git/repository.rb
+ 4
− 12
View file @ a4e75e7a
@@ -898,12 +898,8 @@ module Gitlab
end
 
def fetch_source_branch!(source_repository, source_branch, local_ref)
Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled|
if is_enabled
gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
else
rugged_fetch_source_branch(source_repository, source_branch, local_ref)
end
wrapped_gitaly_errors do
gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
end
end
 
@@ -1058,12 +1054,8 @@ module Gitlab
end
 
def bundle_to_disk(save_path)
gitaly_migrate(:bundle_to_disk) do |is_enabled|
if is_enabled
gitaly_repository_client.create_bundle(save_path)
else
run_git!(%W(bundle create #{save_path} --all))
end
wrapped_gitaly_errors do
gitaly_repository_client.create_bundle(save_path)
end
 
true
lib/gitlab/shell.rb
+ 3
− 41
View file @ a4e75e7a
@@ -92,21 +92,13 @@ module Gitlab
# Ex.
# import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
def import_repository(storage, name, url)
if url.start_with?('.', '/')
raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
end
 
relative_path = "#{name}.git"
cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
GitalyGitlabProjects.new(storage, relative_path)
else
# The timeout ensures the subprocess won't hang forever
gitlab_projects(storage, relative_path)
end
end
cmd = GitalyGitlabProjects.new(storage, relative_path)
 
success = cmd.import_project(url, git_timeout)
raise Error, cmd.output unless success
@@ -126,12 +118,8 @@ module Gitlab
# fetch_remote(my_repo, "upstream")
#
def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
gitaly_migrate(:fetch_remote) do |is_enabled|
if is_enabled
repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune)
else
local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
end
wrapped_gitaly_errors do
repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune)
end
end
 
@@ -389,28 +377,6 @@ module Gitlab
)
end
 
def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
vars = { force: forced, tags: !no_tags, prune: prune }
if ssh_auth&.ssh_import?
if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
vars[:ssh_key] = ssh_auth.ssh_private_key
end
if ssh_auth.ssh_known_hosts.present?
vars[:known_hosts] = ssh_auth.ssh_known_hosts
end
end
cmd = gitlab_projects(storage_name, repository_relative_path)
success = cmd.fetch_remote(remote, git_timeout, vars)
raise Error, cmd.output unless success
success
end
def gitlab_shell_fast_execute(cmd)
output, status = gitlab_shell_fast_execute_helper(cmd)
 
@@ -440,10 +406,6 @@ module Gitlab
Gitlab.config.gitlab_shell.git_timeout
end
 
def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
wrapped_gitaly_errors { Gitlab::GitalyClient.migrate(method, status: status, &block) }
end
def wrapped_gitaly_errors
yield
rescue GRPC::NotFound, GRPC::BadStatus => e
spec/lib/gitlab/git/repository_spec.rb
+ 31
− 39
View file @ a4e75e7a
@@ -1716,59 +1716,51 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
 
describe '#fetch_source_branch!' do
shared_examples '#fetch_source_branch!' do
let(:local_ref) { 'refs/merge-requests/1/head' }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') }
after do
ensure_seeds
end
let(:local_ref) { 'refs/merge-requests/1/head' }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') }
 
context 'when the branch exists' do
context 'when the commit does not exist locally' do
let(:source_branch) { 'new-branch-for-fetch-source-branch' }
let(:source_rugged) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { source_repository.rugged } }
let(:new_oid) { new_commit_edit_old_file(source_rugged).oid }
after do
ensure_seeds
end
 
before do
source_rugged.branches.create(source_branch, new_oid)
end
context 'when the branch exists' do
context 'when the commit does not exist locally' do
let(:source_branch) { 'new-branch-for-fetch-source-branch' }
let(:source_rugged) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { source_repository.rugged } }
let(:new_oid) { new_commit_edit_old_file(source_rugged).oid }
 
it 'writes the ref' do
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
expect(repository.commit(local_ref).sha).to eq(new_oid)
end
before do
source_rugged.branches.create(source_branch, new_oid)
end
 
context 'when the commit exists locally' do
let(:source_branch) { 'master' }
let(:expected_oid) { SeedRepo::LastCommit::ID }
it 'writes the ref' do
# Sanity check: the commit should already exist
expect(repository.commit(expected_oid)).not_to be_nil
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
expect(repository.commit(local_ref).sha).to eq(expected_oid)
end
it 'writes the ref' do
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
expect(repository.commit(local_ref).sha).to eq(new_oid)
end
end
 
context 'when the branch does not exist' do
let(:source_branch) { 'definitely-not-master' }
context 'when the commit exists locally' do
let(:source_branch) { 'master' }
let(:expected_oid) { SeedRepo::LastCommit::ID }
it 'writes the ref' do
# Sanity check: the commit should already exist
expect(repository.commit(expected_oid)).not_to be_nil
 
it 'does not write the ref' do
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(false)
expect(repository.commit(local_ref)).to be_nil
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
expect(repository.commit(local_ref).sha).to eq(expected_oid)
end
end
end
 
it_behaves_like '#fetch_source_branch!'
context 'when the branch does not exist' do
let(:source_branch) { 'definitely-not-master' }
 
context 'without gitaly', :skip_gitaly_mock do
it_behaves_like '#fetch_source_branch!'
it 'does not write the ref' do
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(false)
expect(repository.commit(local_ref)).to be_nil
end
end
end
 
spec/lib/gitlab/shell_spec.rb
+ 24
− 81
View file @ a4e75e7a
@@ -403,46 +403,36 @@ describe Gitlab::Shell do
end
 
describe '#create_repository' do
shared_examples '#create_repository' do
let(:repository_storage) { 'default' }
let(:repository_storage_path) do
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
end
end
let(:repo_name) { 'project/path' }
let(:created_path) { File.join(repository_storage_path, repo_name + '.git') }
after do
FileUtils.rm_rf(created_path)
let(:repository_storage) { 'default' }
let(:repository_storage_path) do
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
end
end
let(:repo_name) { 'project/path' }
let(:created_path) { File.join(repository_storage_path, repo_name + '.git') }
 
it 'creates a repository' do
expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_truthy
expect(File.stat(created_path).mode & 0o777).to eq(0o770)
after do
FileUtils.rm_rf(created_path)
end
 
hooks_path = File.join(created_path, 'hooks')
expect(File.lstat(hooks_path)).to be_symlink
expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path)
end
it 'creates a repository' do
expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_truthy
 
it 'returns false when the command fails' do
FileUtils.mkdir_p(File.dirname(created_path))
# This file will block the creation of the repo's .git directory. That
# should cause #create_repository to fail.
FileUtils.touch(created_path)
expect(File.stat(created_path).mode & 0o777).to eq(0o770)
 
expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_falsy
end
hooks_path = File.join(created_path, 'hooks')
expect(File.lstat(hooks_path)).to be_symlink
expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path)
end
 
context 'with gitaly' do
it_behaves_like '#create_repository'
end
it 'returns false when the command fails' do
FileUtils.mkdir_p(File.dirname(created_path))
# This file will block the creation of the repo's .git directory. That
# should cause #create_repository to fail.
FileUtils.touch(created_path)
 
context 'without gitaly', :skip_gitaly_mock do
it_behaves_like '#create_repository'
expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_falsy
end
end
 
@@ -513,22 +503,12 @@ describe Gitlab::Shell do
end
end
 
shared_examples 'fetch_remote' do |gitaly_on|
describe '#fetch_remote' do
def fetch_remote(ssh_auth = nil, prune = true)
gitlab_shell.fetch_remote(repository.raw_repository, 'remote-name', ssh_auth: ssh_auth, prune: prune)
end
 
def expect_gitlab_projects(fail = false, options = {})
expect(gitlab_projects).to receive(:fetch_remote).with(
'remote-name',
timeout,
options
).and_return(!fail)
allow(gitlab_projects).to receive(:output).and_return('error') if fail
end
def expect_gitaly_call(fail, options = {})
def expect_call(fail, options = {})
receive_fetch_remote =
if fail
receive(:fetch_remote).and_raise(GRPC::NotFound)
@@ -539,16 +519,6 @@ describe Gitlab::Shell do
expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive_fetch_remote
end
 
if gitaly_on
def expect_call(fail, options = {})
expect_gitaly_call(fail, options)
end
else
def expect_call(fail, options = {})
expect_gitlab_projects(fail, options)
end
end
def build_ssh_auth(opts = {})
defaults = {
ssh_import?: true,
@@ -634,14 +604,6 @@ describe Gitlab::Shell do
expect(fetch_remote(ssh_auth)).to be_truthy
end
end
end
describe '#fetch_remote local', :skip_gitaly_mock do
it_should_behave_like 'fetch_remote', false
end
describe '#fetch_remote gitaly' do
it_should_behave_like 'fetch_remote', true
 
context 'gitaly call' do
let(:remote_name) { 'remote-name' }
@@ -683,25 +645,6 @@ describe Gitlab::Shell do
end.to raise_error(Gitlab::Shell::Error, "error")
end
end
context 'without gitaly', :disable_gitaly do
it 'returns true when the command succeeds' do
expect(gitlab_projects).to receive(:import_project).with(import_url, timeout) { true }
result = gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url)
expect(result).to be_truthy
end
it 'raises an exception when the command fails' do
allow(gitlab_projects).to receive(:output) { 'error' }
expect(gitlab_projects).to receive(:import_project) { false }
expect do
gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url)
end.to raise_error(Gitlab::Shell::Error, "error")
end
end
end
end
 
spec/lib/gitlab/workhorse_spec.rb
+ 11
− 13
View file @ a4e75e7a
@@ -36,22 +36,20 @@ describe Gitlab::Workhorse do
allow(described_class).to receive(:git_archive_cache_disabled?).and_return(cache_disabled)
end
 
context 'when Gitaly workhorse_archive feature is enabled' do
it 'sets the header correctly' do
key, command, params = decode_workhorse_header(subject)
it 'sets the header correctly' do
key, command, params = decode_workhorse_header(subject)
 
expect(key).to eq('Gitlab-Workhorse-Send-Data')
expect(command).to eq('git-archive')
expect(params).to include(gitaly_params)
end
expect(key).to eq('Gitlab-Workhorse-Send-Data')
expect(command).to eq('git-archive')
expect(params).to include(gitaly_params)
end
 
context 'when archive caching is disabled' do
let(:cache_disabled) { true }
context 'when archive caching is disabled' do
let(:cache_disabled) { true }
 
it 'tells workhorse not to use the cache' do
_, _, params = decode_workhorse_header(subject)
expect(params).to include({ 'DisableCache' => true })
end
it 'tells workhorse not to use the cache' do
_, _, params = decode_workhorse_header(subject)
expect(params).to include({ 'DisableCache' => true })
end
end
 
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment