diff --git a/CHANGELOG b/CHANGELOG
index 764df7cf55a87c7e45abbe2c1fb3982ed7c8d734..6b23ce2b27d017010f01849798ed719136bd7ff8 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -3,6 +3,7 @@ Please view this file on the master branch, on stable branches it's out of date.
v 8.10.0 (unreleased)
- Fix commit builds API, return all builds for all pipelines for given commit. !4849
- Replace Haml with Hamlit to make view rendering faster. !3666
+ - Refactor repository paths handling to allow multiple git mount points
- Wrap code blocks on Activies and Todos page. !4783 (winniehell)
- Align flash messages with left side of page content !4959 (winniehell)
- Display last commit of deleted branch in push events !4699 (winniehell)
diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION
index 4a36342fcab700951adb18ae7adc930997f6c3f4..fd2a01863fdd3035fac5918c59666363544bfe23 100644
--- a/GITLAB_SHELL_VERSION
+++ b/GITLAB_SHELL_VERSION
@@ -1 +1 @@
-3.0.0
+3.1.0
diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb
index 26796b08facafab6d946695930bc1a15267e629d..f312a7ccca3618f59f60e139e72af2d5d2eca10f 100644
--- a/app/helpers/projects_helper.rb
+++ b/app/helpers/projects_helper.rb
@@ -327,9 +327,9 @@ module ProjectsHelper
end
end
- def sanitize_repo_path(message)
+ def sanitize_repo_path(project, message)
return '' unless message.present?
- message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]")
+ message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
end
end
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index da19462f2652ffe92182394dd3b45e4b3e091e0b..8b52cc824cd6c860cbf9af349cd7575bdb58170b 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base
delegate :name, to: :owner, allow_nil: true, prefix: true
- after_create :ensure_dir_exist
after_update :move_dir, if: :path_changed?
+
+ # Save the storage paths before the projects are destroyed to use them on after destroy
+ before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths }
after_destroy :rm_dir
scope :root, -> { where('type IS NULL') }
@@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base
owner_name
end
- def ensure_dir_exist
- gitlab_shell.add_namespace(path)
- end
-
- def rm_dir
- # Move namespace directory into trash.
- # We will remove it later async
- new_path = "#{path}+#{id}+deleted"
-
- if gitlab_shell.mv_namespace(path, new_path)
- message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
- Gitlab::AppLogger.info message
-
- # Remove namespace directroy async with delay so
- # GitLab has time to remove all projects first
- GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path)
- end
- end
-
def move_dir
- # Ensure old directory exists before moving it
- gitlab_shell.add_namespace(path_was)
-
if any_project_has_container_registry_tags?
raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
- if gitlab_shell.mv_namespace(path_was, path)
- Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
-
- # If repositories moved successfully we need to
- # send update instructions to users.
- # However we cannot allow rollback since we moved namespace dir
- # So we basically we mute exceptions in next actions
- begin
- send_update_instructions
- rescue
- # Returning false does not rollback after_* transaction but gives
- # us information about failing some of tasks
- false
+ # Move the namespace directory in all storages paths used by member projects
+ repository_storage_paths.each do |repository_storage_path|
+ # Ensure old directory exists before moving it
+ gitlab_shell.add_namespace(repository_storage_path, path_was)
+
+ unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
+ # if we cannot move namespace directory we should rollback
+ # db changes in order to prevent out of sync between db and fs
+ raise Exception.new('namespace directory cannot be moved')
end
- else
- # if we cannot move namespace directory we should rollback
- # db changes in order to prevent out of sync between db and fs
- raise Exception.new('namespace directory cannot be moved')
+ end
+
+ Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
+
+ # If repositories moved successfully we need to
+ # send update instructions to users.
+ # However we cannot allow rollback since we moved namespace dir
+ # So we basically we mute exceptions in next actions
+ begin
+ send_update_instructions
+ rescue
+ # Returning false does not rollback after_* transaction but gives
+ # us information about failing some of tasks
+ false
end
end
@@ -152,4 +138,33 @@ class Namespace < ActiveRecord::Base
def find_fork_of(project)
projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
end
+
+ private
+
+ def repository_storage_paths
+ # We need to get the storage paths for all the projects, even the ones that are
+ # pending delete. Unscoping also get rids of the default order, which causes
+ # problems with SELECT DISTINCT.
+ Project.unscoped do
+ projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
+ end
+ end
+
+ def rm_dir
+ # Remove the namespace directory in all storages paths used by member projects
+ @old_repository_storage_paths.each do |repository_storage_path|
+ # Move namespace directory into trash.
+ # We will remove it later async
+ new_path = "#{path}+#{id}+deleted"
+
+ if gitlab_shell.mv_namespace(repository_storage_path, path, new_path)
+ message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
+ Gitlab::AppLogger.info message
+
+ # Remove namespace directroy async with delay so
+ # GitLab has time to remove all projects first
+ GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
+ end
+ end
+ end
end
diff --git a/app/models/project.rb b/app/models/project.rb
index 73ded09c162ed50b3ba1d71c2cb12bb526730d05..2f6901e0e3ed31a4a8b5659df701438e26fa8144 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -26,6 +26,9 @@ class Project < ActiveRecord::Base
default_value_for :container_registry_enabled, gitlab_config_features.container_registry
default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
+ after_create :ensure_dir_exist
+ after_save :ensure_dir_exist, if: :namespace_id_changed?
+
# set last_activity_at to the same as created_at
after_create :set_last_activity_at
def set_last_activity_at
@@ -165,6 +168,9 @@ class Project < ActiveRecord::Base
validate :visibility_level_allowed_by_group
validate :visibility_level_allowed_as_fork
validate :check_wiki_path_conflict
+ validates :repository_storage,
+ presence: true,
+ inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
add_authentication_token_field :runners_token
before_save :ensure_runners_token
@@ -376,6 +382,10 @@ class Project < ActiveRecord::Base
end
end
+ def repository_storage_path
+ Gitlab.config.repositories.storages[repository_storage]
+ end
+
def team
@team ||= ProjectTeam.new(self)
end
@@ -842,12 +852,12 @@ class Project < ActiveRecord::Base
raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
end
- if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
+ if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
- gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
+ gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
reset_events_cache
@@ -988,7 +998,7 @@ class Project < ActiveRecord::Base
def create_repository
# Forked import is handled asynchronously
unless forked?
- if gitlab_shell.add_repository(path_with_namespace)
+ if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
@@ -1140,4 +1150,8 @@ class Project < ActiveRecord::Base
_, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
status.zero?
end
+
+ def ensure_dir_exist
+ gitlab_shell.add_namespace(repository_storage_path, namespace.path)
+ end
end
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 25d82929c0b1ff454f6f8cb53e5be8f75fdb3a99..a255710f57711952d797d5ae5a8af20f27c0d579 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -159,7 +159,7 @@ class ProjectWiki
private
def init_repo(path_with_namespace)
- gitlab_shell.add_repository(path_with_namespace)
+ gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace)
end
def commit_details(action, message = nil, title = nil)
@@ -173,7 +173,7 @@ class ProjectWiki
end
def path_to_repo
- @path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git")
+ @path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git")
end
def update_project_activity
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 2a6a3b086c200282120f5bf5a0e3c4c67b336d85..f45c3d06abd95fa802dae78d63d87e52582eb776 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -39,7 +39,7 @@ class Repository
# Return absolute path to repository
def path_to_repo
@path_to_repo ||= File.expand_path(
- File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git")
+ File.join(@project.repository_storage_path, path_with_namespace + ".git")
)
end
diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index f09072975c3e9a109b18b00d7103984962dcbeb6..882606e38d0e9c907bfc8e56a1ed0ff65b7575c6 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -51,13 +51,13 @@ module Projects
return true if params[:skip_repo] == true
# There is a possibility project does not have repository or wiki
- return true unless gitlab_shell.exists?(path + '.git')
+ return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
new_path = removal_path(path)
- if gitlab_shell.mv_repository(path, new_path)
+ if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
- GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path)
+ GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path)
else
false
end
diff --git a/app/services/projects/housekeeping_service.rb b/app/services/projects/housekeeping_service.rb
index 43db29315a166cb3b2360765831a7c5e6220ff2d..a47df22f1bacbe51a3f760b5d3de9316effa4b7e 100644
--- a/app/services/projects/housekeeping_service.rb
+++ b/app/services/projects/housekeeping_service.rb
@@ -24,7 +24,7 @@ module Projects
def execute
raise LeaseTaken unless try_obtain_lease
- GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace)
+ GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace)
ensure
Gitlab::Metrics.measure(:reset_pushes_since_gc) do
@project.update_column(:pushes_since_gc, 0)
diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb
index 9159ec089593d122f9168de7c9a62a014cd1c2a2..163ebf26c84393982360b8f7652ae47d989bac3b 100644
--- a/app/services/projects/import_service.rb
+++ b/app/services/projects/import_service.rb
@@ -42,7 +42,7 @@ module Projects
def import_repository
begin
- gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
+ gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
rescue Gitlab::Shell::Error => e
raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
end
diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb
index 03b57dea51e4d4a9c8f83fa3d0ab8306ed19c3df..bc7f8bf433b6ed5f77bfdc8310fb0d0fa92c9b50 100644
--- a/app/services/projects/transfer_service.rb
+++ b/app/services/projects/transfer_service.rb
@@ -50,12 +50,12 @@ module Projects
project.send_move_instructions(old_path)
# Move main repository
- unless gitlab_shell.mv_repository(old_path, new_path)
+ unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path)
raise TransferError.new('Cannot move project')
end
# Move wiki repo also if present
- gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
+ gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki")
# clear project cached events
project.reset_events_cache
diff --git a/app/views/projects/imports/new.html.haml b/app/views/projects/imports/new.html.haml
index a8a8caf728036728c601804cad75f5a6e5f9b80f..2cd8d03e30e9f20c86692304e50620de485f6152 100644
--- a/app/views/projects/imports/new.html.haml
+++ b/app/views/projects/imports/new.html.haml
@@ -10,7 +10,7 @@
.panel-body
%pre
:preserve
- #{sanitize_repo_path(@project.import_error)}
+ #{sanitize_repo_path(@project, @project.import_error)}
= form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
= render "shared/import_form", f: f
diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb
index f3327ca9e61909f117fee35a63ba4daf8c5b1a5b..09035a7cf2daaaef53c1e531d1e8e8d9e9b25888 100644
--- a/app/workers/post_receive.rb
+++ b/app/workers/post_receive.rb
@@ -4,10 +4,10 @@ class PostReceive
sidekiq_options queue: :post_receive
def perform(repo_path, identifier, changes)
- if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s)
- repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "")
+ if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
+ repo_path.gsub!(path[1].to_s, "")
else
- log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"")
+ log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
end
post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes)
diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb
index d947f1055160b7f4ae23d6d1c42ee8d74e17ea25..f7604e48f8320778277c4140d3d10629502f20a4 100644
--- a/app/workers/repository_fork_worker.rb
+++ b/app/workers/repository_fork_worker.rb
@@ -12,7 +12,7 @@ class RepositoryForkWorker
return
end
- result = gitlab_shell.fork_repository(source_path, target_path)
+ result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path)
unless result
logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
project.mark_import_as_failed('The project could not be forked.')
diff --git a/config/gitlab.teatro.yml b/config/gitlab.teatro.yml
index 01c8dc5ff98e2cd278ccea0a1146131a69958137..75b79b837e0c75a3dd9c1fa3bd9b10fae294fa86 100644
--- a/config/gitlab.teatro.yml
+++ b/config/gitlab.teatro.yml
@@ -47,11 +47,13 @@ production: &base
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
+ repositories:
+ storages: # REPO PATHS MUST NOT BE A SYMLINK!!!
+ default: /apps/repositories/
+
gitlab_shell:
path: /apps/gitlab-shell/
- # REPOS_PATH MUST NOT BE A SYMLINK!!!
- repos_path: /apps/repositories/
hooks_path: /apps/gitlab-shell/hooks/
upload_pack: true
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 75e1a3c1093343ffe062df4efa988c8f0603d86f..325eca72862841ab5c8d251edebfba41388b485c 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -428,6 +428,13 @@ production: &base
satellites:
path: /home/git/gitlab-satellites/
+ ## Repositories settings
+ repositories:
+ # Paths where repositories can be stored. Give the canonicalized absolute pathname.
+ # NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
+ storages: # You must have at least a `default` storage path.
+ default: /home/git/repositories/
+
## Backup settings
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
@@ -452,9 +459,6 @@ production: &base
## GitLab Shell settings
gitlab_shell:
path: /home/git/gitlab-shell/
-
- # REPOS_PATH MUST NOT BE A SYMLINK!!!
- repos_path: /home/git/repositories/
hooks_path: /home/git/gitlab-shell/hooks/
# File that contains the secret key for verifying access for gitlab-shell.
@@ -528,11 +532,13 @@ test:
# user: YOUR_USERNAME
satellites:
path: tmp/tests/gitlab-satellites/
+ repositories:
+ storages:
+ default: tmp/tests/repositories/
backup:
path: tmp/tests/backups
gitlab_shell:
path: tmp/tests/gitlab-shell/
- repos_path: tmp/tests/repositories/
hooks_path: tmp/tests/gitlab-shell/hooks/
issues_tracker:
redmine:
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index c6dc1e4ab38da2b2a162c878665d82ef7bbbc1e9..a93996cec72921f3120c9a9dac768632623959ea 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -304,13 +304,20 @@ Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitla
Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret')
Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil?
Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil?
-Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/'
Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host
Settings.gitlab_shell['ssh_port'] ||= 22
Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user
Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user
Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix)
+#
+# Repositories
+#
+Settings['repositories'] ||= Settingslogic.new({})
+Settings.repositories['storages'] ||= {}
+# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0
+Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/'
+
#
# Backup
#
diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb
new file mode 100644
index 0000000000000000000000000000000000000000..3ba9e36c567c13aaacd8018524052358cbb7c0f5
--- /dev/null
+++ b/config/initializers/6_validations.rb
@@ -0,0 +1,24 @@
+def storage_name_valid?(name)
+ !!(name =~ /\A[a-zA-Z0-9\-_]+\z/)
+end
+
+def find_parent_path(name, path)
+ Gitlab.config.repositories.storages.detect do |n, p|
+ name != n && path.chomp('/').start_with?(p.chomp('/'))
+ end
+end
+
+def error(message)
+ raise "#{message}. Please fix this in your gitlab.yml before starting GitLab."
+end
+
+error('No repository storage path defined') if Gitlab.config.repositories.storages.empty?
+
+Gitlab.config.repositories.storages.each do |name, path|
+ error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name)
+
+ parent_name, _parent_path = find_parent_path(name, path)
+ if parent_name
+ error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages")
+ end
+end
diff --git a/db/migrate/20160608195742_add_repository_storage_to_projects.rb b/db/migrate/20160608195742_add_repository_storage_to_projects.rb
new file mode 100644
index 0000000000000000000000000000000000000000..c700d2b569d2d074997a700cad33d28eaf2fb903
--- /dev/null
+++ b/db/migrate/20160608195742_add_repository_storage_to_projects.rb
@@ -0,0 +1,12 @@
+class AddRepositoryStorageToProjects < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+ disable_ddl_transaction!
+
+ def up
+ add_column_with_default(:projects, :repository_storage, :string, default: 'default')
+ end
+
+ def down
+ remove_column(:projects, :repository_storage)
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 7a8377f687c00aede850a1901fcd3414abdb3646..a7173116b1b50099311d2d66a9985fed0cb5f601 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -796,38 +796,39 @@ ActiveRecord::Schema.define(version: 20160620115026) do
t.datetime "created_at"
t.datetime "updated_at"
t.integer "creator_id"
- t.boolean "issues_enabled", default: true, null: false
- t.boolean "merge_requests_enabled", default: true, null: false
- t.boolean "wiki_enabled", default: true, null: false
+ t.boolean "issues_enabled", default: true, null: false
+ t.boolean "merge_requests_enabled", default: true, null: false
+ t.boolean "wiki_enabled", default: true, null: false
t.integer "namespace_id"
- t.boolean "snippets_enabled", default: true, null: false
+ t.boolean "snippets_enabled", default: true, null: false
t.datetime "last_activity_at"
t.string "import_url"
- t.integer "visibility_level", default: 0, null: false
- t.boolean "archived", default: false, null: false
+ t.integer "visibility_level", default: 0, null: false
+ t.boolean "archived", default: false, null: false
t.string "avatar"
t.string "import_status"
t.float "repository_size", default: 0.0
- t.integer "star_count", default: 0, null: false
+ t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
t.integer "commit_count", default: 0
t.text "import_error"
t.integer "ci_id"
- t.boolean "builds_enabled", default: true, null: false
- t.boolean "shared_runners_enabled", default: true, null: false
+ t.boolean "builds_enabled", default: true, null: false
+ t.boolean "shared_runners_enabled", default: true, null: false
t.string "runners_token"
t.string "build_coverage_regex"
- t.boolean "build_allow_git_fetch", default: true, null: false
- t.integer "build_timeout", default: 3600, null: false
+ t.boolean "build_allow_git_fetch", default: true, null: false
+ t.integer "build_timeout", default: 3600, null: false
t.boolean "pending_delete", default: false
- t.boolean "public_builds", default: true, null: false
+ t.boolean "public_builds", default: true, null: false
t.integer "pushes_since_gc", default: 0
t.boolean "last_repository_check_failed"
t.datetime "last_repository_check_at"
t.boolean "container_registry_enabled"
- t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
+ t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "has_external_issue_tracker"
+ t.string "repository_storage", default: "default", null: false
end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
diff --git a/doc/README.md b/doc/README.md
index be0d17084c7f6dcaac3cb6f3c6fb96356a848c30..b98d6812a81556af24b59e8f9cb67d38f3d82d76 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -34,6 +34,7 @@
- [Operations](operations/README.md) Keeping GitLab up and running.
- [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects.
- [Repository checks](administration/repository_checks.md) Periodic Git repository checks.
+- [Repository storages](administration/repository_storages.md) Manage the paths used to store repositories.
- [Security](security/README.md) Learn what you can do to further secure your GitLab instance.
- [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed.
- [Update](update/README.md) Update guides to upgrade your installation.
diff --git a/doc/administration/repository_storages.md b/doc/administration/repository_storages.md
new file mode 100644
index 0000000000000000000000000000000000000000..81bfe173151eb183a42993e3eb6fefb59c6533c1
--- /dev/null
+++ b/doc/administration/repository_storages.md
@@ -0,0 +1,18 @@
+# Repository storages
+
+GitLab allows you to define repository storage paths to enable distribution of
+storage load between several mount points.
+
+## For installations from source
+
+Add your repository storage paths in your `gitlab.yml` under repositories -> storages, using key -> value pairs.
+
+>**Notes:**
+- You must have at least one storage path called `default`.
+- In order for backups to work correctly the storage path must **not** be a
+mount point and the GitLab user should have correct permissions for the parent
+directory of the path.
+
+## For omnibus installations
+
+Follow the instructions at https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/configuration.md#storing-git-data-in-an-alternative-directory
diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md
index 8a38937062e003b658d54d081539f00029735543..2b305cb5c991f7cb83eed7c37d171509e9e3ad4c 100644
--- a/doc/raketasks/import.md
+++ b/doc/raketasks/import.md
@@ -14,7 +14,8 @@
- For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed
it in the `/etc/gitlab/gitlab.rb` file.
- For installations from source, it is usually located at: `/home/git/repositories` or you can see where
-your repositories are located by looking at `config/gitlab.yml` under the `gitlab_shell => repos_path` entry.
+your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries
+(you'll usually use the `default` storage path to start).
New folder needs to have git user ownership and read/write/execute access for git user and its group:
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 1d361569d595c623b4ea178df6c0905e2f9b0216..b32503e8516b7608eae864a99e46be97d89818db 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -20,6 +20,20 @@ module API
@wiki ||= params[:project].end_with?('.wiki') &&
!Project.find_with_namespace(params[:project])
end
+
+ def project
+ @project ||= begin
+ project_path = params[:project]
+
+ # Check for *.wiki repositories.
+ # Strip out the .wiki from the pathname before finding the
+ # project. This applies the correct project permissions to
+ # the wiki repository as well.
+ project_path.chomp!('.wiki') if wiki?
+
+ Project.find_with_namespace(project_path)
+ end
+ end
end
post "/allowed" do
@@ -32,16 +46,6 @@ module API
User.find_by(id: params[:user_id])
end
- project_path = params[:project]
-
- # Check for *.wiki repositories.
- # Strip out the .wiki from the pathname before finding the
- # project. This applies the correct project permissions to
- # the wiki repository as well.
- project_path.chomp!('.wiki') if wiki?
-
- project = Project.find_with_namespace(project_path)
-
access =
if wiki?
Gitlab::GitAccessWiki.new(actor, project)
@@ -49,7 +53,17 @@ module API
Gitlab::GitAccess.new(actor, project)
end
- access.check(params[:action], params[:changes])
+ access_status = access.check(params[:action], params[:changes])
+
+ response = { status: access_status.status, message: access_status.message }
+
+ if access_status.status
+ # Return the repository full path so that gitlab-shell has it when
+ # handling ssh commands
+ response[:repository_path] = project.repository.path_to_repo
+ end
+
+ response
end
#
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 7b91215d50b6f17118917404ae5a01089bdd5bd1..b9773f98d752a7c8ea4a077c387f8b3776a1bd57 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -2,8 +2,6 @@ require 'yaml'
module Backup
class Repository
- attr_reader :repos_path
-
def dump
prepare
@@ -50,10 +48,12 @@ module Backup
end
def restore
- if File.exists?(repos_path)
+ Gitlab.config.repositories.storages.each do |name, path|
+ next unless File.exists?(path)
+
# Move repos dir to 'repositories.old' dir
- bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s)
- FileUtils.mv(repos_path, bk_repos_path)
+ bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s)
+ FileUtils.mv(path, bk_repos_path)
end
FileUtils.mkdir_p(repos_path)
@@ -61,7 +61,7 @@ module Backup
Project.find_each(batch_size: 1000) do |project|
$progress.print " * #{project.path_with_namespace} ... "
- project.namespace.ensure_dir_exist if project.namespace
+ project.ensure_dir_exist
if File.exists?(path_to_bundle(project))
FileUtils.mkdir_p(path_to_repo(project))
@@ -100,8 +100,8 @@ module Backup
end
$progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
- cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks"
- if system(cmd)
+ cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
+ if system(*cmd)
$progress.puts " [DONE]".color(:green)
else
puts " [FAILED]".color(:red)
@@ -120,10 +120,6 @@ module Backup
File.join(backup_repos_path, project.path_with_namespace + ".bundle")
end
- def repos_path
- Gitlab.config.gitlab_shell.repos_path
- end
-
def backup_repos_path
File.join(Gitlab.config.backup.path, "repositories")
end
@@ -139,5 +135,11 @@ module Backup
def silent
{err: '/dev/null', out: '/dev/null'}
end
+
+ private
+
+ def repository_storage_paths_args
+ Gitlab.config.repositories.storages.values
+ end
end
end
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 3e3986d63827811c894e741869c09f1fadb64da1..e31840ef9190451ba8004d0194142cdd235a2d57 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -18,77 +18,82 @@ module Gitlab
# Init new repository
#
+ # storage - project's storage path
# name - project path with namespace
#
# Ex.
- # add_repository("gitlab/gitlab-ci")
+ # add_repository("/path/to/storage", "gitlab/gitlab-ci")
#
- def add_repository(name)
+ def add_repository(storage, name)
Gitlab::Utils.system_silent([gitlab_shell_projects_path,
- 'add-project', "#{name}.git"])
+ 'add-project', storage, "#{name}.git"])
end
# Import repository
#
+ # storage - project's storage path
# name - project path with namespace
#
# Ex.
- # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git")
+ # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
#
- def import_repository(name, url)
- output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900'])
+ def import_repository(storage, name, url)
+ output, status = Popen::popen([gitlab_shell_projects_path, 'import-project',
+ storage, "#{name}.git", url, '900'])
raise Error, output unless status.zero?
true
end
# Move repository
- #
+ # storage - project's storage path
# path - project path with namespace
# new_path - new project path with namespace
#
# Ex.
- # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new")
+ # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
#
- def mv_repository(path, new_path)
+ def mv_repository(storage, path, new_path)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project',
- "#{path}.git", "#{new_path}.git"])
+ storage, "#{path}.git", "#{new_path}.git"])
end
# Fork repository to new namespace
- #
+ # storage - project's storage path
# path - project path with namespace
# fork_namespace - namespace for forked project
#
# Ex.
- # fork_repository("gitlab/gitlab-ci", "randx")
+ # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx")
#
- def fork_repository(path, fork_namespace)
+ def fork_repository(storage, path, fork_namespace)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project',
- "#{path}.git", fork_namespace])
+ storage, "#{path}.git", fork_namespace])
end
# Remove repository from file system
#
+ # storage - project's storage path
# name - project path with namespace
#
# Ex.
- # remove_repository("gitlab/gitlab-ci")
+ # remove_repository("/path/to/storage", "gitlab/gitlab-ci")
#
- def remove_repository(name)
+ def remove_repository(storage, name)
Gitlab::Utils.system_silent([gitlab_shell_projects_path,
- 'rm-project', "#{name}.git"])
+ 'rm-project', storage, "#{name}.git"])
end
# Gc repository
#
+ # storage - project storage path
# path - project path with namespace
#
# Ex.
- # gc("gitlab/gitlab-ci")
+ # gc("/path/to/storage", "gitlab/gitlab-ci")
#
- def gc(path)
+ def gc(storage, path)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc',
- "#{path}.git"])
+ storage, "#{path}.git"])
end
# Add new key to gitlab-shell
@@ -133,31 +138,31 @@ module Gitlab
# Add empty directory for storing repositories
#
# Ex.
- # add_namespace("gitlab")
+ # add_namespace("/path/to/storage", "gitlab")
#
- def add_namespace(name)
- FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name)
+ def add_namespace(storage, name)
+ FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name)
end
# Remove directory from repositories storage
# Every repository inside this directory will be removed too
#
# Ex.
- # rm_namespace("gitlab")
+ # rm_namespace("/path/to/storage", "gitlab")
#
- def rm_namespace(name)
- FileUtils.rm_r(full_path(name), force: true)
+ def rm_namespace(storage, name)
+ FileUtils.rm_r(full_path(storage, name), force: true)
end
# Move namespace directory inside repositories storage
#
# Ex.
- # mv_namespace("gitlab", "gitlabhq")
+ # mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
#
- def mv_namespace(old_name, new_name)
- return false if exists?(new_name) || !exists?(old_name)
+ def mv_namespace(storage, old_name, new_name)
+ return false if exists?(storage, new_name) || !exists?(storage, old_name)
- FileUtils.mv(full_path(old_name), full_path(new_name))
+ FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
end
def url_to_repo(path)
@@ -176,11 +181,11 @@ module Gitlab
# Check if such directory exists in repositories.
#
# Usage:
- # exists?('gitlab')
- # exists?('gitlab/cookies.git')
+ # exists?(storage, 'gitlab')
+ # exists?(storage, 'gitlab/cookies.git')
#
- def exists?(dir_name)
- File.exist?(full_path(dir_name))
+ def exists?(storage, dir_name)
+ File.exist?(full_path(storage, dir_name))
end
protected
@@ -193,14 +198,10 @@ module Gitlab
File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
end
- def repos_path
- Gitlab.config.gitlab_shell.repos_path
- end
-
- def full_path(dir_name)
+ def full_path(storage, dir_name)
raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
- File.join(repos_path, dir_name)
+ File.join(storage, dir_name)
end
def gitlab_shell_projects_path
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 2286ac8829c9813ca17e60670e0aecc3b85046b9..730978d502b501b613c59299cda2beca9145f85f 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -167,7 +167,7 @@ module Gitlab
def import_wiki
unless project.wiki_enabled?
wiki = WikiFormatter.new(project)
- gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url)
+ gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
project.update_attribute(:wiki_enabled, true)
end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 12d6ac45fb6d0a75008f35b6cc7306b12a6c51d0..e9a4e37ec484b9063ee0d4aa6b3d3eab5b5d0305 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -356,97 +356,108 @@ namespace :gitlab do
########################
def check_repo_base_exists
- print "Repo base directory exists? ... "
+ puts "Repo base directory exists?"
- repo_base_path = Gitlab.config.gitlab_shell.repos_path
+ Gitlab.config.repositories.storages.each do |name, repo_base_path|
+ print "#{name}... "
- if File.exists?(repo_base_path)
- puts "yes".color(:green)
- else
- puts "no".color(:red)
- puts "#{repo_base_path} is missing".color(:red)
- try_fixing_it(
- "This should have been created when setting up GitLab Shell.",
- "Make sure it's set correctly in config/gitlab.yml",
- "Make sure GitLab Shell is installed correctly."
- )
- for_more_information(
- see_installation_guide_section "GitLab Shell"
- )
- fix_and_rerun
+ if File.exists?(repo_base_path)
+ puts "yes".color(:green)
+ else
+ puts "no".color(:red)
+ puts "#{repo_base_path} is missing".color(:red)
+ try_fixing_it(
+ "This should have been created when setting up GitLab Shell.",
+ "Make sure it's set correctly in config/gitlab.yml",
+ "Make sure GitLab Shell is installed correctly."
+ )
+ for_more_information(
+ see_installation_guide_section "GitLab Shell"
+ )
+ fix_and_rerun
+ end
end
end
def check_repo_base_is_not_symlink
- print "Repo base directory is a symlink? ... "
+ puts "Repo storage directories are symlinks?"
- repo_base_path = Gitlab.config.gitlab_shell.repos_path
- unless File.exists?(repo_base_path)
- puts "can't check because of previous errors".color(:magenta)
- return
- end
+ Gitlab.config.repositories.storages.each do |name, repo_base_path|
+ print "#{name}... "
- unless File.symlink?(repo_base_path)
- puts "no".color(:green)
- else
- puts "yes".color(:red)
- try_fixing_it(
- "Make sure it's set to the real directory in config/gitlab.yml"
- )
- fix_and_rerun
+ unless File.exists?(repo_base_path)
+ puts "can't check because of previous errors".color(:magenta)
+ return
+ end
+
+ unless File.symlink?(repo_base_path)
+ puts "no".color(:green)
+ else
+ puts "yes".color(:red)
+ try_fixing_it(
+ "Make sure it's set to the real directory in config/gitlab.yml"
+ )
+ fix_and_rerun
+ end
end
end
def check_repo_base_permissions
- print "Repo base access is drwxrws---? ... "
+ puts "Repo paths access is drwxrws---?"
- repo_base_path = Gitlab.config.gitlab_shell.repos_path
- unless File.exists?(repo_base_path)
- puts "can't check because of previous errors".color(:magenta)
- return
- end
+ Gitlab.config.repositories.storages.each do |name, repo_base_path|
+ print "#{name}... "
- if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
- puts "yes".color(:green)
- else
- puts "no".color(:red)
- try_fixing_it(
- "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
- "sudo chmod -R ug-s #{repo_base_path}",
- "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
- )
- for_more_information(
- see_installation_guide_section "GitLab Shell"
- )
- fix_and_rerun
+ unless File.exists?(repo_base_path)
+ puts "can't check because of previous errors".color(:magenta)
+ return
+ end
+
+ if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
+ puts "yes".color(:green)
+ else
+ puts "no".color(:red)
+ try_fixing_it(
+ "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
+ "sudo chmod -R ug-s #{repo_base_path}",
+ "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
+ )
+ for_more_information(
+ see_installation_guide_section "GitLab Shell"
+ )
+ fix_and_rerun
+ end
end
end
def check_repo_base_user_and_group
gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
- print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... "
+ puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
- repo_base_path = Gitlab.config.gitlab_shell.repos_path
- unless File.exists?(repo_base_path)
- puts "can't check because of previous errors".color(:magenta)
- return
- end
+ Gitlab.config.repositories.storages.each do |name, repo_base_path|
+ print "#{name}... "
- uid = uid_for(gitlab_shell_ssh_user)
- gid = gid_for(gitlab_shell_owner_group)
- if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
- puts "yes".color(:green)
- else
- puts "no".color(:red)
- puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
- try_fixing_it(
- "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
- )
- for_more_information(
- see_installation_guide_section "GitLab Shell"
- )
- fix_and_rerun
+ unless File.exists?(repo_base_path)
+ puts "can't check because of previous errors".color(:magenta)
+ return
+ end
+
+ uid = uid_for(gitlab_shell_ssh_user)
+ gid = gid_for(gitlab_shell_owner_group)
+ if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
+ puts "yes".color(:green)
+ else
+ puts "no".color(:red)
+ puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
+ try_fixing_it(
+ "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
+ )
+ for_more_information(
+ see_installation_guide_section "GitLab Shell"
+ )
+ fix_and_rerun
+ end
end
end
@@ -473,7 +484,7 @@ namespace :gitlab do
else
puts "wrong or missing hooks".color(:red)
try_fixing_it(
- sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"),
+ sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"),
'Check the hooks_path in config/gitlab.yml',
'Check your gitlab-shell installation'
)
@@ -785,13 +796,13 @@ namespace :gitlab do
namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab"
task check: :environment do
- namespace_dirs = Dir.glob(
- File.join(Gitlab.config.gitlab_shell.repos_path, '*')
- )
+ Gitlab.config.repositories.storages.each do |name, path|
+ namespace_dirs = Dir.glob(File.join(path, '*'))
- namespace_dirs.each do |namespace_dir|
- repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
- repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
+ namespace_dirs.each do |namespace_dir|
+ repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
+ repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
+ end
end
end
end
@@ -799,12 +810,12 @@ namespace :gitlab do
namespace :user do
desc "GitLab | Check the integrity of a specific user's repositories"
task :check_repos, [:username] => :environment do |t, args|
- username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue))
+ username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
user = User.find_by(username: username)
if user
repo_dirs = user.authorized_projects.map do |p|
File.join(
- Gitlab.config.gitlab_shell.repos_path,
+ p.repository_storage_path,
"#{p.path_with_namespace}.git"
)
end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index ab0028d6603013ad73ef92fdf97ac643282e6fdc..b7cbdc6cd78f2901cf7aeee2d3b8771886cf793f 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -5,36 +5,36 @@ namespace :gitlab do
warn_user_is_not_gitlab
remove_flag = ENV['REMOVE']
-
namespaces = Namespace.pluck(:path)
- git_base_path = Gitlab.config.gitlab_shell.repos_path
- all_dirs = Dir.glob(git_base_path + '/*')
+ Gitlab.config.repositories.storages.each do |name, git_base_path|
+ all_dirs = Dir.glob(git_base_path + '/*')
- puts git_base_path.color(:yellow)
- puts "Looking for directories to remove... "
+ puts git_base_path.color(:yellow)
+ puts "Looking for directories to remove... "
- all_dirs.reject! do |dir|
- # skip if git repo
- dir =~ /.git$/
- end
+ all_dirs.reject! do |dir|
+ # skip if git repo
+ dir =~ /.git$/
+ end
- all_dirs.reject! do |dir|
- dir_name = File.basename dir
+ all_dirs.reject! do |dir|
+ dir_name = File.basename dir
- # skip if namespace present
- namespaces.include?(dir_name)
- end
+ # skip if namespace present
+ namespaces.include?(dir_name)
+ end
- all_dirs.each do |dir_path|
+ all_dirs.each do |dir_path|
- if remove_flag
- if FileUtils.rm_rf dir_path
- puts "Removed...#{dir_path}".color(:red)
+ if remove_flag
+ if FileUtils.rm_rf dir_path
+ puts "Removed...#{dir_path}".color(:red)
+ else
+ puts "Cannot remove #{dir_path}".color(:red)
+ end
else
- puts "Cannot remove #{dir_path}".color(:red)
+ puts "Can be removed: #{dir_path}".color(:red)
end
- else
- puts "Can be removed: #{dir_path}".color(:red)
end
end
@@ -48,20 +48,21 @@ namespace :gitlab do
warn_user_is_not_gitlab
move_suffix = "+orphaned+#{Time.now.to_i}"
- repo_root = Gitlab.config.gitlab_shell.repos_path
- # Look for global repos (legacy, depth 1) and normal repos (depth 2)
- IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
- find.each_line do |path|
- path.chomp!
- repo_with_namespace = path.
- sub(repo_root, '').
- sub(%r{^/*}, '').
- chomp('.git').
- chomp('.wiki')
- next if Project.find_with_namespace(repo_with_namespace)
- new_path = path + move_suffix
- puts path.inspect + ' -> ' + new_path.inspect
- File.rename(path, new_path)
+ Gitlab.config.repositories.storages.each do |name, repo_root|
+ # Look for global repos (legacy, depth 1) and normal repos (depth 2)
+ IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
+ find.each_line do |path|
+ path.chomp!
+ repo_with_namespace = path.
+ sub(repo_root, '').
+ sub(%r{^/*}, '').
+ chomp('.git').
+ chomp('.wiki')
+ next if Project.find_with_namespace(repo_with_namespace)
+ new_path = path + move_suffix
+ puts path.inspect + ' -> ' + new_path.inspect
+ File.rename(path, new_path)
+ end
end
end
end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index 4753f00c26ab76512a3033c939e04e6e2a49bb8b..dbdd4e977e8becc230551f32abbda26a691ab8b6 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -2,73 +2,73 @@ namespace :gitlab do
namespace :import do
# How to use:
#
- # 1. copy the bare repos under the repos_path (commonly /home/git/repositories)
+ # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories)
# 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production
#
# Notes:
# * The project owner will set to the first administator of the system
# * Existing projects will be skipped
#
- desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance"
+ desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
task repos: :environment do
+ Gitlab.config.repositories.storages.each do |name, git_base_path|
+ repos_to_import = Dir.glob(git_base_path + '/**/*.git')
- git_base_path = Gitlab.config.gitlab_shell.repos_path
- repos_to_import = Dir.glob(git_base_path + '/**/*.git')
+ repos_to_import.each do |repo_path|
+ # strip repo base path
+ repo_path[0..git_base_path.length] = ''
- repos_to_import.each do |repo_path|
- # strip repo base path
- repo_path[0..git_base_path.length] = ''
+ path = repo_path.sub(/\.git$/, '')
+ group_name, name = File.split(path)
+ group_name = nil if group_name == '.'
- path = repo_path.sub(/\.git$/, '')
- group_name, name = File.split(path)
- group_name = nil if group_name == '.'
+ puts "Processing #{repo_path}".color(:yellow)
- puts "Processing #{repo_path}".color(:yellow)
-
- if path.end_with?('.wiki')
- puts " * Skipping wiki repo"
- next
- end
+ if path.end_with?('.wiki')
+ puts " * Skipping wiki repo"
+ next
+ end
- project = Project.find_with_namespace(path)
+ project = Project.find_with_namespace(path)
- if project
- puts " * #{project.name} (#{repo_path}) exists"
- else
- user = User.admins.reorder("id").first
+ if project
+ puts " * #{project.name} (#{repo_path}) exists"
+ else
+ user = User.admins.reorder("id").first
- project_params = {
- name: name,
- path: name
- }
+ project_params = {
+ name: name,
+ path: name
+ }
- # find group namespace
- if group_name
- group = Namespace.find_by(path: group_name)
- # create group namespace
- unless group
- group = Group.new(:name => group_name)
- group.path = group_name
- group.owner = user
- if group.save
- puts " * Created Group #{group.name} (#{group.id})".color(:green)
- else
- puts " * Failed trying to create group #{group.name}".color(:red)
+ # find group namespace
+ if group_name
+ group = Namespace.find_by(path: group_name)
+ # create group namespace
+ unless group
+ group = Group.new(:name => group_name)
+ group.path = group_name
+ group.owner = user
+ if group.save
+ puts " * Created Group #{group.name} (#{group.id})".color(:green)
+ else
+ puts " * Failed trying to create group #{group.name}".color(:red)
+ end
end
+ # set project group
+ project_params[:namespace_id] = group.id
end
- # set project group
- project_params[:namespace_id] = group.id
- end
- project = Projects::CreateService.new(user, project_params).execute
+ project = Projects::CreateService.new(user, project_params).execute
- if project.persisted?
- puts " * Created #{project.name} (#{repo_path})".color(:green)
- project.update_repository_size
- project.update_commit_count
- else
- puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
- puts " Errors: #{project.errors.messages}".color(:red)
+ if project.persisted?
+ puts " * Created #{project.name} (#{repo_path})".color(:green)
+ project.update_repository_size
+ project.update_commit_count
+ else
+ puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
+ puts " Errors: #{project.errors.messages}".color(:red)
+ end
end
end
end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 352b566df247a5a58f53ee9647df945777a114c4..fe43d40e6d23dac528ca1b3dc8f8185a049f7e00 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -62,7 +62,10 @@ namespace :gitlab do
puts ""
puts "GitLab Shell".color(:yellow)
puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
- puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}"
+ puts "Repository storage paths:"
+ Gitlab.config.repositories.storages.each do |name, path|
+ puts "- #{name}: \t#{path}"
+ end
puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
puts "Git:\t\t#{Gitlab.config.git.bin_path}"
diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake
index c7596e7abcb72d33b1403f84c37a4f43dcf8362c..ffcc76e549891f1ff13051f35d146c968b08abfe 100644
--- a/lib/tasks/gitlab/list_repos.rake
+++ b/lib/tasks/gitlab/list_repos.rake
@@ -9,7 +9,7 @@ namespace :gitlab do
scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids)
end
scope.find_each do |project|
- base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace)
+ base = File.join(project.repository_storage_path, project.path_with_namespace)
puts base + '.git'
puts base + '.wiki.git'
end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index b1648a4602abab48320fdb1a0350017b722df1e9..263798e9c2261449e4bdc8f4d194b53c2b58b0f3 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -12,7 +12,6 @@ namespace :gitlab do
gitlab_url = Gitlab.config.gitlab.url
# gitlab-shell requires a / at the end of the url
gitlab_url += '/' unless gitlab_url.end_with?('/')
- repos_path = Gitlab.config.gitlab_shell.repos_path
target_dir = Gitlab.config.gitlab_shell.path
# Clone if needed
@@ -35,7 +34,6 @@ namespace :gitlab do
user: user,
gitlab_url: gitlab_url,
http_settings: {self_signed_cert: false}.stringify_keys,
- repos_path: repos_path,
auth_file: File.join(home_dir, ".ssh", "authorized_keys"),
redis: {
bin: %x{which redis-cli}.chomp,
@@ -58,10 +56,10 @@ namespace :gitlab do
File.open("config.yml", "w+") {|f| f.puts config.to_yaml}
# Launch installation process
- system(*%W(bin/install))
+ system(*%W(bin/install) + repository_storage_paths_args)
# (Re)create hooks
- system(*%W(bin/create-hooks))
+ system(*%W(bin/create-hooks) + repository_storage_paths_args)
end
# Required for debian packaging with PKGR: Setup .ssh/environment with
@@ -87,7 +85,8 @@ namespace :gitlab do
if File.exists?(path_to_repo)
print '-'
else
- if Gitlab::Shell.new.add_repository(project.path_with_namespace)
+ if Gitlab::Shell.new.add_repository(project.repository_storage_path,
+ project.path_with_namespace)
print '.'
else
print 'F'
@@ -138,4 +137,3 @@ namespace :gitlab do
system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
end
end
-
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
index d0c019044b7eed695b8a82d2a810a799894e492f..ab96b1d35932c9802787ef65e6db2f189961993e 100644
--- a/lib/tasks/gitlab/task_helpers.rake
+++ b/lib/tasks/gitlab/task_helpers.rake
@@ -125,10 +125,16 @@ namespace :gitlab do
end
def all_repos
- IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
- find.each_line do |path|
- yield path.chomp
+ Gitlab.config.repositories.storages.each do |name, path|
+ IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
+ find.each_line do |path|
+ yield path.chomp
+ end
end
end
end
+
+ def repository_storage_paths_args
+ Gitlab.config.repositories.storages.values
+ end
end
diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb
index 09e0bbfd00b807f702bd336cc980a396975f56bb..604204cca0a4c3a6ce760694bd4101d50990fde0 100644
--- a/spec/helpers/projects_helper_spec.rb
+++ b/spec/helpers/projects_helper_spec.rb
@@ -123,11 +123,17 @@ describe ProjectsHelper do
end
describe '#sanitized_import_error' do
+ let(:project) { create(:project) }
+
+ before do
+ allow(project).to receive(:repository_storage_path).and_return('/base/repo/path')
+ end
+
it 'removes the repo path' do
- repo = File.join(Gitlab.config.gitlab_shell.repos_path, '/namespace/test.git')
+ repo = '/base/repo/path/namespace/test.git'
import_error = "Could not clone #{repo}\n"
- expect(sanitize_repo_path(import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')
+ expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')
end
end
end
diff --git a/spec/initializers/6_validations_spec.rb b/spec/initializers/6_validations_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..5178bd130f4c1748935693ddbae1266abf5f9448
--- /dev/null
+++ b/spec/initializers/6_validations_spec.rb
@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe '6_validations', lib: true do
+ context 'with correct settings' do
+ before do
+ mock_storages('foo' => '/a/b/c', 'bar' => 'a/b/d')
+ end
+
+ it 'passes through' do
+ expect { load_validations }.not_to raise_error
+ end
+ end
+
+ context 'with invalid storage names' do
+ before do
+ mock_storages('name with spaces' => '/a/b/c')
+ end
+
+ it 'throws an error' do
+ expect { load_validations }.to raise_error('"name with spaces" is not a valid storage name. Please fix this in your gitlab.yml before starting GitLab.')
+ end
+ end
+
+ context 'with nested storage paths' do
+ before do
+ mock_storages('foo' => '/a/b/c', 'bar' => '/a/b/c/d')
+ end
+
+ it 'throws an error' do
+ expect { load_validations }.to raise_error('bar is a nested path of foo. Nested paths are not supported for repository storages. Please fix this in your gitlab.yml before starting GitLab.')
+ end
+ end
+
+ def mock_storages(storages)
+ allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+ end
+
+ def load_validations
+ load File.join(__dir__, '../../config/initializers/6_validations.rb')
+ end
+end
diff --git a/spec/lib/gitlab/backend/shell_spec.rb b/spec/lib/gitlab/backend/shell_spec.rb
index fd869f48b5c223890d58c5f85b86219212b986fb..e15f13f985b9831fe7ee2c6eeeac716b3ff45e6e 100644
--- a/spec/lib/gitlab/backend/shell_spec.rb
+++ b/spec/lib/gitlab/backend/shell_spec.rb
@@ -13,6 +13,11 @@ describe Gitlab::Shell, lib: true do
it { is_expected.to respond_to :add_repository }
it { is_expected.to respond_to :remove_repository }
it { is_expected.to respond_to :fork_repository }
+ it { is_expected.to respond_to :gc }
+ it { is_expected.to respond_to :add_namespace }
+ it { is_expected.to respond_to :rm_namespace }
+ it { is_expected.to respond_to :mv_namespace }
+ it { is_expected.to respond_to :exists? }
it { expect(gitlab_shell.url_to_repo('diaspora')).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "diaspora.git") }
diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb
index 4e68ac5e63af9e0e9894a28732a76bde4ed147e9..cbea407f9bff89feee999517517c501555a2d16c 100644
--- a/spec/models/namespace_spec.rb
+++ b/spec/models/namespace_spec.rb
@@ -57,6 +57,7 @@ describe Namespace, models: true do
describe :move_dir do
before do
@namespace = create :namespace
+ @project = create :project, namespace: @namespace
allow(@namespace).to receive(:path_changed?).and_return(true)
end
@@ -87,8 +88,13 @@ describe Namespace, models: true do
end
describe :rm_dir do
- it "should remove dir" do
- expect(namespace.rm_dir).to be_truthy
+ let!(:project) { create(:project, namespace: namespace) }
+ let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.path) }
+
+ before { namespace.destroy }
+
+ it "should remove its dirs when deleted" do
+ expect(File.exist?(path)).to be(false)
end
end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index d305cd9ff1e39e4a77a065c932af9bfc1a3d79fe..ee1142fa3aac58654bd5375b3ebb3f73d929803b 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -56,6 +56,7 @@ describe Project, models: true do
it { is_expected.to validate_length_of(:description).is_within(0..2000) }
it { is_expected.to validate_presence_of(:creator) }
it { is_expected.to validate_presence_of(:namespace) }
+ it { is_expected.to validate_presence_of(:repository_storage) }
it 'should not allow new projects beyond user limits' do
project2 = build(:project)
@@ -84,6 +85,20 @@ describe Project, models: true do
end
end
end
+
+ context 'repository storages inclussion' do
+ let(:project2) { build(:project, repository_storage: 'missing') }
+
+ before do
+ storages = { 'custom' => 'tmp/tests/custom_repositories' }
+ allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+ end
+
+ it "should not allow repository storages that don't match a label in the configuration" do
+ expect(project2).not_to be_valid
+ expect(project2.errors[:repository_storage].first).to match(/is not included in the list/)
+ end
+ end
end
describe 'default_scope' do
@@ -131,6 +146,24 @@ describe Project, models: true do
end
end
+ describe '#repository_storage_path' do
+ let(:project) { create(:project, repository_storage: 'custom') }
+
+ before do
+ FileUtils.mkdir('tmp/tests/custom_repositories')
+ storages = { 'custom' => 'tmp/tests/custom_repositories' }
+ allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+ end
+
+ after do
+ FileUtils.rm_rf('tmp/tests/custom_repositories')
+ end
+
+ it 'returns the repository storage path' do
+ expect(project.repository_storage_path).to eq('tmp/tests/custom_repositories')
+ end
+ end
+
it 'should return valid url to repo' do
project = Project.new(path: 'somewhere')
expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git')
@@ -729,12 +762,12 @@ describe Project, models: true do
expect(gitlab_shell).to receive(:mv_repository).
ordered.
- with("#{ns}/foo", "#{ns}/#{project.path}").
+ with(project.repository_storage_path, "#{ns}/foo", "#{ns}/#{project.path}").
and_return(true)
expect(gitlab_shell).to receive(:mv_repository).
ordered.
- with("#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki").
+ with(project.repository_storage_path, "#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki").
and_return(true)
expect_any_instance_of(SystemHooksService).
@@ -826,7 +859,7 @@ describe Project, models: true do
context 'using a regular repository' do
it 'creates the repository' do
expect(shell).to receive(:add_repository).
- with(project.path_with_namespace).
+ with(project.repository_storage_path, project.path_with_namespace).
and_return(true)
expect(project.repository).to receive(:after_create)
@@ -836,7 +869,7 @@ describe Project, models: true do
it 'adds an error if the repository could not be created' do
expect(shell).to receive(:add_repository).
- with(project.path_with_namespace).
+ with(project.repository_storage_path, project.path_with_namespace).
and_return(false)
expect(project.repository).not_to receive(:after_create)
diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb
index 437c89c35776ab35a7424a8bd58cd5253072bb6d..fcea45f19bad486070cd0f8ea58e64d5075859e4 100644
--- a/spec/requests/api/internal_spec.rb
+++ b/spec/requests/api/internal_spec.rb
@@ -72,6 +72,7 @@ describe API::API, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy
+ expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
end
end
@@ -81,6 +82,7 @@ describe API::API, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy
+ expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
end
end
end
diff --git a/spec/services/destroy_group_service_spec.rb b/spec/services/destroy_group_service_spec.rb
index afa89b84175a778dbc10224f686288676aaf8878..eca8ddd8ea4bc4d2da1d96fdf0b0ef6c6039a6ea 100644
--- a/spec/services/destroy_group_service_spec.rb
+++ b/spec/services/destroy_group_service_spec.rb
@@ -23,8 +23,8 @@ describe DestroyGroupService, services: true do
Sidekiq::Testing.inline! { destroy_group(group, user) }
end
- it { expect(gitlab_shell.exists?(group.path)).to be_falsey }
- it { expect(gitlab_shell.exists?(remove_path)).to be_falsey }
+ it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey }
+ it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey }
end
context 'Sidekiq fake' do
@@ -33,8 +33,8 @@ describe DestroyGroupService, services: true do
Sidekiq::Testing.fake! { destroy_group(group, user) }
end
- it { expect(gitlab_shell.exists?(group.path)).to be_falsey }
- it { expect(gitlab_shell.exists?(remove_path)).to be_truthy }
+ it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey }
+ it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_truthy }
end
end
diff --git a/spec/services/projects/housekeeping_service_spec.rb b/spec/services/projects/housekeeping_service_spec.rb
index 4c5ced7e746a29bb00fe8bcbd9533c9b30ef39c2..bd4dc6a0f7989d3c2815991b8a04230d53e0c1a2 100644
--- a/spec/services/projects/housekeeping_service_spec.rb
+++ b/spec/services/projects/housekeeping_service_spec.rb
@@ -12,7 +12,7 @@ describe Projects::HousekeepingService do
it 'enqueues a sidekiq job' do
expect(subject).to receive(:try_obtain_lease).and_return(true)
- expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.path_with_namespace)
+ expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.repository_storage_path, project.path_with_namespace)
subject.execute
expect(project.pushes_since_gc).to eq(0)
diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb
index 068c9a1219c984f127524b7004d8843c00cc9975..d5d4d7c56ef5d492f7a03a861b4797d0113c9cdd 100644
--- a/spec/services/projects/import_service_spec.rb
+++ b/spec/services/projects/import_service_spec.rb
@@ -36,7 +36,7 @@ describe Projects::ImportService, services: true do
end
it 'succeeds if repository import is successfully' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
result = subject.execute
@@ -44,7 +44,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if repository import fails' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository'))
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository'))
result = subject.execute
@@ -64,7 +64,7 @@ describe Projects::ImportService, services: true do
end
it 'succeeds if importer succeeds' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(true)
result = subject.execute
@@ -74,7 +74,7 @@ describe Projects::ImportService, services: true do
it 'flushes various caches' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).
- with(project.path_with_namespace, project.import_url).
+ with(project.repository_storage_path, project.path_with_namespace, project.import_url).
and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).
@@ -90,7 +90,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if importer fails' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(false)
result = subject.execute
@@ -100,7 +100,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if importer raise an error' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API'))
result = subject.execute
diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb
index 426bf53f19852563f5e3a64c46b2e4148bdde72c..be5331e4770a35f205beaa2640f4473e3b5a86d3 100644
--- a/spec/support/test_env.rb
+++ b/spec/support/test_env.rb
@@ -80,8 +80,9 @@ module TestEnv
end
def setup_gitlab_shell
- unless File.directory?(Rails.root.join(*%w(tmp tests gitlab-shell)))
- `rake gitlab:shell:install`
+ unless File.directory?(Gitlab.config.gitlab_shell.path)
+ # TODO: Remove `[shards]` when gitlab-shell v3.1.0 is published
+ `rake gitlab:shell:install[shards]`
end
end
@@ -127,14 +128,14 @@ module TestEnv
def copy_repo(project)
base_repo_path = File.expand_path(factory_repo_path_bare)
- target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git")
+ target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git")
FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{base_repo_path}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path
end
def repos_path
- Gitlab.config.gitlab_shell.repos_path
+ Gitlab.config.repositories.storages.default
end
def backup_path
@@ -143,7 +144,7 @@ module TestEnv
def copy_forked_repo_with_submodules(project)
base_repo_path = File.expand_path(forked_repo_path_bare)
- target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git")
+ target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git")
FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{base_repo_path}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path
diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 25da0917134cdb819157f77327580da9780da22a..02308530d13db6459da27bdb833ab5c452df6878 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -98,67 +98,107 @@ describe 'gitlab:app namespace rake task' do
@backup_tar = tars_glob.first
end
- before do
- create_backup
- end
-
- after do
- FileUtils.rm(@backup_tar)
- end
+ context 'tar creation' do
+ before do
+ create_backup
+ end
- context 'archive file permissions' do
- it 'should set correct permissions on the tar file' do
- expect(File.exist?(@backup_tar)).to be_truthy
- expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600')
+ after do
+ FileUtils.rm(@backup_tar)
end
- context 'with custom archive_permissions' do
- before do
- allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651)
- # We created a backup in a before(:all) so it got the default permissions.
- # We now need to do some work to create a _new_ backup file using our stub.
- FileUtils.rm(@backup_tar)
- create_backup
+ context 'archive file permissions' do
+ it 'should set correct permissions on the tar file' do
+ expect(File.exist?(@backup_tar)).to be_truthy
+ expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600')
end
- it 'uses the custom permissions' do
- expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651')
+ context 'with custom archive_permissions' do
+ before do
+ allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651)
+ # We created a backup in a before(:all) so it got the default permissions.
+ # We now need to do some work to create a _new_ backup file using our stub.
+ FileUtils.rm(@backup_tar)
+ create_backup
+ end
+
+ it 'uses the custom permissions' do
+ expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651')
+ end
end
end
- end
- it 'should set correct permissions on the tar contents' do
- tar_contents, exit_status = Gitlab::Popen.popen(
- %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz}
- )
- expect(exit_status).to eq(0)
- expect(tar_contents).to match('db/')
- expect(tar_contents).to match('uploads.tar.gz')
- expect(tar_contents).to match('repositories/')
- expect(tar_contents).to match('builds.tar.gz')
- expect(tar_contents).to match('artifacts.tar.gz')
- expect(tar_contents).to match('lfs.tar.gz')
- expect(tar_contents).to match('registry.tar.gz')
- expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/)
- end
+ it 'should set correct permissions on the tar contents' do
+ tar_contents, exit_status = Gitlab::Popen.popen(
+ %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz}
+ )
+ expect(exit_status).to eq(0)
+ expect(tar_contents).to match('db/')
+ expect(tar_contents).to match('uploads.tar.gz')
+ expect(tar_contents).to match('repositories/')
+ expect(tar_contents).to match('builds.tar.gz')
+ expect(tar_contents).to match('artifacts.tar.gz')
+ expect(tar_contents).to match('lfs.tar.gz')
+ expect(tar_contents).to match('registry.tar.gz')
+ expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/)
+ end
- it 'should delete temp directories' do
- temp_dirs = Dir.glob(
- File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}')
- )
+ it 'should delete temp directories' do
+ temp_dirs = Dir.glob(
+ File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}')
+ )
+
+ expect(temp_dirs).to be_empty
+ end
- expect(temp_dirs).to be_empty
+ context 'registry disabled' do
+ let(:enable_registry) { false }
+
+ it 'should not create registry.tar.gz' do
+ tar_contents, exit_status = Gitlab::Popen.popen(
+ %W{tar -tvf #{@backup_tar}}
+ )
+ expect(exit_status).to eq(0)
+ expect(tar_contents).not_to match('registry.tar.gz')
+ end
+ end
end
- context 'registry disabled' do
- let(:enable_registry) { false }
+ context 'multiple repository storages' do
+ let(:project_a) { create(:project, repository_storage: 'default') }
+ let(:project_b) { create(:project, repository_storage: 'custom') }
+
+ before do
+ FileUtils.mkdir('tmp/tests/default_storage')
+ FileUtils.mkdir('tmp/tests/custom_storage')
+ storages = {
+ 'default' => 'tmp/tests/default_storage',
+ 'custom' => 'tmp/tests/custom_storage'
+ }
+ allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+
+ # Create the projects now, after mocking the settings but before doing the backup
+ project_a
+ project_b
+
+ # We only need a backup of the repositories for this test
+ ENV["SKIP"] = "db,uploads,builds,artifacts,lfs,registry"
+ create_backup
+ end
+
+ after do
+ FileUtils.rm_rf('tmp/tests/default_storage')
+ FileUtils.rm_rf('tmp/tests/custom_storage')
+ FileUtils.rm(@backup_tar)
+ end
- it 'should not create registry.tar.gz' do
+ it 'should include repositories in all repository storages' do
tar_contents, exit_status = Gitlab::Popen.popen(
- %W{tar -tvf #{@backup_tar}}
+ %W{tar -tvf #{@backup_tar} repositories}
)
expect(exit_status).to eq(0)
- expect(tar_contents).not_to match('registry.tar.gz')
+ expect(tar_contents).to match("repositories/#{project_a.path_with_namespace}.bundle")
+ expect(tar_contents).to match("repositories/#{project_b.path_with_namespace}.bundle")
end
end
end # backup_create task
diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb
index b8e73682c91ca61910d3f1c1674de82a546a39c5..20b1a343c27887d7d7121652dc5e3ed3dfd976b6 100644
--- a/spec/workers/post_receive_spec.rb
+++ b/spec/workers/post_receive_spec.rb
@@ -91,6 +91,6 @@ describe PostReceive do
end
def pwd(project)
- File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace)
+ File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace)
end
end
diff --git a/spec/workers/repository_fork_worker_spec.rb b/spec/workers/repository_fork_worker_spec.rb
index 4ef05eb29d2b3f1c95f7b97bd7ed92325b0aa2ec..5f762282b5ea7ddae6a61adc14e006fe1c8780e3 100644
--- a/spec/workers/repository_fork_worker_spec.rb
+++ b/spec/workers/repository_fork_worker_spec.rb
@@ -14,6 +14,7 @@ describe RepositoryForkWorker do
describe "#perform" do
it "creates a new repository from a fork" do
expect(shell).to receive(:fork_repository).with(
+ project.repository_storage_path,
project.path_with_namespace,
fork_project.namespace.path
).and_return(true)
@@ -25,9 +26,11 @@ describe RepositoryForkWorker do
end
it 'flushes various caches' do
- expect(shell).to receive(:fork_repository).
- with(project.path_with_namespace, fork_project.namespace.path).
- and_return(true)
+ expect(shell).to receive(:fork_repository).with(
+ project.repository_storage_path,
+ project.path_with_namespace,
+ fork_project.namespace.path
+ ).and_return(true)
expect_any_instance_of(Repository).to receive(:expire_emptiness_caches).
and_call_original