Refactor repository paths handling to allow multiple git mount points

CHANGELOG

@@ -79,6 +79,7 @@ v 8.9.0 (unreleased)
   - Allow users to create confidential issues in private projects
   - Measure CPU time for instrumented methods
   - Instrument private methods and private instance methods by default instead just public methods
+  - Refactor repository paths handling to allow multiple git mount points
 v 8.8.5 (unreleased)
   - Ensure branch cleanup regardless of whether the GitHub import process succeeds

GITLAB_SHELL_VERSION

-3.0.0
+3.1.0

app/helpers/projects_helper.rb

@@ -323,9 +323,9 @@ module ProjectsHelper
     end
   end
-  def sanitize_repo_path(message)
+  def sanitize_repo_path(project, message)
     return '' unless message.present?
-    message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]")
+    message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
   end
 end

app/models/namespace.rb

@@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base
   delegate :name, to: :owner, allow_nil: true, prefix: true
-  after_create :ensure_dir_exist
   after_update :move_dir, if: :path_changed?
+
+  # Save the storage paths before the projects are destroyed to use them on after destroy
+  before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths }
   after_destroy :rm_dir
   scope :root, -> { where('type IS NULL') }
@@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base
     owner_name
   end
-  def ensure_dir_exist
-    gitlab_shell.add_namespace(path)
-  end
-
-  def rm_dir
-    # Move namespace directory into trash.
-    # We will remove it later async
-    new_path = "#{path}+#{id}+deleted"
-
-    if gitlab_shell.mv_namespace(path, new_path)
-      message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
-      Gitlab::AppLogger.info message
-
-      # Remove namespace directroy async with delay so
-      # GitLab has time to remove all projects first
-      GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path)
-    end
-  end
-
   def move_dir
-    # Ensure old directory exists before moving it
-    gitlab_shell.add_namespace(path_was)
-
     if any_project_has_container_registry_tags?
       raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
     end
-    if gitlab_shell.mv_namespace(path_was, path)
-      Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
-
-      # If repositories moved successfully we need to
-      # send update instructions to users.
-      # However we cannot allow rollback since we moved namespace dir
-      # So we basically we mute exceptions in next actions
-      begin
-        send_update_instructions
-      rescue
-        # Returning false does not rollback after_* transaction but gives
-        # us information about failing some of tasks
-        false
+    # Move the namespace directory in all storages paths used by member projects
+    repository_storage_paths.each do |repository_storage_path|
+      # Ensure old directory exists before moving it
+      gitlab_shell.add_namespace(repository_storage_path, path_was)
+
+      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
+        # if we cannot move namespace directory we should rollback
+        # db changes in order to prevent out of sync between db and fs
+        raise Exception.new('namespace directory cannot be moved')
       end
-    else
-      # if we cannot move namespace directory we should rollback
-      # db changes in order to prevent out of sync between db and fs
-      raise Exception.new('namespace directory cannot be moved')
+    end
+
+    Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
+
+    # If repositories moved successfully we need to
+    # send update instructions to users.
+    # However we cannot allow rollback since we moved namespace dir
+    # So we basically we mute exceptions in next actions
+    begin
+      send_update_instructions
+    rescue
+      # Returning false does not rollback after_* transaction but gives
+      # us information about failing some of tasks
+      false
     end
   end
@@ -152,4 +138,28 @@ class Namespace < ActiveRecord::Base
   def find_fork_of(project)
     projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
   end
+
+  private
+
+  def repository_storage_paths
+    projects.to_a.map(&:repository_storage_path).uniq
+  end
+
+  def rm_dir
+    # Remove the namespace directory in all storages paths used by member projects
+    @old_repository_storage_paths.each do |repository_storage_path|
+      # Move namespace directory into trash.
+      # We will remove it later async
+      new_path = "#{path}+#{id}+deleted"
+
+      if gitlab_shell.mv_namespace(repository_storage_path, path, new_path)
+        message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
+        Gitlab::AppLogger.info message
+
+        # Remove namespace directroy async with delay so
+        # GitLab has time to remove all projects first
+        GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
+      end
+    end
+  end
 end

app/models/project.rb

@@ -26,6 +26,15 @@ class Project < ActiveRecord::Base
   default_value_for :container_registry_enabled, gitlab_config_features.container_registry
   default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
+  # TODO: It used to be a namespace task to ensure the namespace directory
+  # existance, but since now projects might be in different paths (shards) they
+  # must account for that. However, checking the directory exists every time the
+  # project is saved might be inefficient. Consider moving this logic to another
+  # moment, or perhaps not to assume the directory exists and have gitlab-shell
+  # take that into consideration (doing `mkdir` himself before each operation,
+  # for example)
+  after_save :ensure_dir_exist
+
   # set last_activity_at to the same as created_at
   after_create :set_last_activity_at
   def set_last_activity_at
@@ -351,6 +360,14 @@ class Project < ActiveRecord::Base
     end
   end
+  def repository_storage
+    super || 'default'
+  end
+
+  def repository_storage_path
+    Gitlab.config.repositories.storages[repository_storage]
+  end
+
   def team
     @team ||= ProjectTeam.new(self)
   end
@@ -803,12 +820,12 @@ class Project < ActiveRecord::Base
       raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
     end
-    if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
+    if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
       # If repository moved successfully we need to send update instructions to users.
       # However we cannot allow rollback since we moved repository
       # So we basically we mute exceptions in next actions
       begin
-        gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
+        gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
         send_move_instructions(old_path_with_namespace)
         reset_events_cache
@@ -949,7 +966,7 @@ class Project < ActiveRecord::Base
   def create_repository
     # Forked import is handled asynchronously
     unless forked?
-      if gitlab_shell.add_repository(path_with_namespace)
+      if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
         repository.after_create
         true
       else
@@ -1078,4 +1095,8 @@ class Project < ActiveRecord::Base
   ensure
     @errors = original_errors
   end
+
+  def ensure_dir_exist
+    gitlab_shell.add_namespace(repository_storage_path, namespace.path)
+  end
 end

app/models/project_wiki.rb

@@ -159,7 +159,7 @@ class ProjectWiki
   private
   def init_repo(path_with_namespace)
-    gitlab_shell.add_repository(path_with_namespace)
+    gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace)
   end
   def commit_details(action, message = nil, title = nil)
@@ -173,7 +173,7 @@ class ProjectWiki
   end
   def path_to_repo
-    @path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git")
+    @path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git")
   end
   def update_project_activity

app/models/repository.rb

@@ -39,7 +39,7 @@ class Repository
   # Return absolute path to repository
   def path_to_repo
     @path_to_repo ||= File.expand_path(
-      File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git")
+      File.join(@project.repository_storage_path, path_with_namespace + ".git")
     )
   end

app/services/projects/destroy_service.rb

@@ -51,13 +51,13 @@ module Projects
       return true if params[:skip_repo] == true
       # There is a possibility project does not have repository or wiki
-      return true unless gitlab_shell.exists?(path + '.git')
+      return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
       new_path = removal_path(path)
-      if gitlab_shell.mv_repository(path, new_path)
+      if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
         log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
-        GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path)
+        GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path)
       else
         false
       end

app/services/projects/housekeeping_service.rb

@@ -24,7 +24,7 @@ module Projects
     def execute
       raise LeaseTaken unless try_obtain_lease
-      GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace)
+      GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace)
     ensure
       Gitlab::Metrics.measure(:reset_pushes_since_gc) do
         @project.update_column(:pushes_since_gc, 0)

app/services/projects/import_service.rb

@@ -37,7 +37,7 @@ module Projects
     def import_repository
       begin
-        gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
+        gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
       rescue Gitlab::Shell::Error => e
         raise Error,  "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
       end

app/services/projects/transfer_service.rb

@@ -50,12 +50,12 @@ module Projects
         project.send_move_instructions(old_path)
         # Move main repository
-        unless gitlab_shell.mv_repository(old_path, new_path)
+        unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path)
           raise TransferError.new('Cannot move project')
         end
         # Move wiki repo also if present
-        gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
+        gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki")
         # clear project cached events
         project.reset_events_cache

app/views/projects/imports/new.html.haml

@@ -10,7 +10,7 @@
     .panel-body
       %pre
         :preserve
-          #{sanitize_repo_path(@project.import_error)}
+          #{sanitize_repo_path(@project, @project.import_error)}
 = form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
   = render "shared/import_form", f: f

app/workers/post_receive.rb

@@ -4,10 +4,10 @@ class PostReceive
   sidekiq_options queue: :post_receive
   def perform(repo_path, identifier, changes)
-    if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s)
-      repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "")
+    if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
+      repo_path.gsub!(path[1].to_s, "")
     else
-      log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"")
+      log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
     end
     post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes)

app/workers/repository_fork_worker.rb

@@ -12,7 +12,7 @@ class RepositoryForkWorker
       return
     end
-    result = gitlab_shell.fork_repository(source_path, target_path)
+    result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path)
     unless result
       logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
       project.mark_import_as_failed('The project could not be forked.')

config/gitlab.teatro.yml

@@ -47,11 +47,13 @@ production: &base
   backup:
     path: "tmp/backups"   # Relative paths are relative to Rails.root (default: tmp/backups/)
+  repositories:
+    storages: # REPO PATHS MUST NOT BE A SYMLINK!!!
+      default: /apps/repositories/
+
   gitlab_shell:
     path: /apps/gitlab-shell/
-    # REPOS_PATH MUST NOT BE A SYMLINK!!!
-    repos_path: /apps/repositories/
     hooks_path: /apps/gitlab-shell/hooks/
     upload_pack: true

config/gitlab.yml.example

@@ -428,6 +428,12 @@ production: &base
   satellites:
     path: /home/git/gitlab-satellites/
+  ## Repositories settings
+  repositories:
+    # Paths where repositories can be stored. Give the canonicalized absolute pathname.
+    # NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
+    storages: # You must have at least a `default` storage path.
+
   ## Backup settings
   backup:
     path: "tmp/backups"   # Relative paths are relative to Rails.root (default: tmp/backups/)
@@ -452,9 +458,6 @@ production: &base
   ## GitLab Shell settings
   gitlab_shell:
     path: /home/git/gitlab-shell/
-
-    # REPOS_PATH MUST NOT BE A SYMLINK!!!
-    repos_path: /home/git/repositories/
     hooks_path: /home/git/gitlab-shell/hooks/
     # File that contains the secret key for verifying access for gitlab-shell.
@@ -528,11 +531,13 @@ test:
     # user: YOUR_USERNAME
   satellites:
     path: tmp/tests/gitlab-satellites/
+  repositories:
+    storages:
+      default: tmp/tests/repositories/
   backup:
     path: tmp/tests/backups
   gitlab_shell:
     path: tmp/tests/gitlab-shell/
-    repos_path: tmp/tests/repositories/
     hooks_path: tmp/tests/gitlab-shell/hooks/
   issues_tracker:
     redmine:

config/initializers/1_settings.rb

@@ -292,6 +292,11 @@ Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
+#
+# Repositories
+#
+Settings.repositories.storages['default'] ||= Settings.gitlab['user_home'] + '/repositories/'
+
 #
 # GitLab Shell
 #
@@ -301,7 +306,6 @@ Settings.gitlab_shell['hooks_path']   ||= Settings.gitlab['user_home'] + '/gitla
 Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret')
 Settings.gitlab_shell['receive_pack']   = true if Settings.gitlab_shell['receive_pack'].nil?
 Settings.gitlab_shell['upload_pack']    = true if Settings.gitlab_shell['upload_pack'].nil?
-Settings.gitlab_shell['repos_path']   ||= Settings.gitlab['user_home'] + '/repositories/'
 Settings.gitlab_shell['ssh_host']     ||= Settings.gitlab.ssh_host
 Settings.gitlab_shell['ssh_port']     ||= 22
 Settings.gitlab_shell['ssh_user']     ||= Settings.gitlab.user

db/migrate/20160608195742_add_repository_storage_to_projects.rb

+class AddRepositoryStorageToProjects < ActiveRecord::Migration
+  def change
+    add_column :projects, :repository_storage, :string
+  end
+end

db/schema.rb

@@ -779,6 +779,7 @@ ActiveRecord::Schema.define(version: 20160610301627) do
     t.boolean  "last_repository_check_failed"
     t.datetime "last_repository_check_at"
     t.boolean  "container_registry_enabled"
+    t.string   "repository_storage"
     t.boolean  "only_allow_merge_if_build_succeeds", default: false,    null: false
     t.boolean  "has_external_issue_tracker"
   end

doc/raketasks/import.md

@@ -14,7 +14,8 @@
 - For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed
 it in the `/etc/gitlab/gitlab.rb` file.
 - For installations from source, it is usually located at: `/home/git/repositories` or you can see where
-your repositories are located by looking at `config/gitlab.yml` under the `gitlab_shell => repos_path` entry.
+your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries
+(you'll usually use the `default` storage path to start).
 New folder needs to have git user ownership and read/write/execute access for git user and its group: