Refactor repository paths handling to allow multiple git mount points

CHANGELOG

@@ -3,6 +3,7 @@ Please view this file on the master branch, on stable branches it's out of date.
 v 8.10.0 (unreleased)
   - Fix commit builds API, return all builds for all pipelines for given commit. !4849
   - Replace Haml with Hamlit to make view rendering faster. !3666
+  - Refactor repository paths handling to allow multiple git mount points
   - Wrap code blocks on Activies and Todos page. !4783 (winniehell)
   - Align flash messages with left side of page content !4959 (winniehell)
   - Display last commit of deleted branch in push events !4699 (winniehell)

GITLAB_SHELL_VERSION

-3.0.0
+3.1.0

app/helpers/projects_helper.rb

@@ -327,9 +327,9 @@ module ProjectsHelper
     end
   end
-  def sanitize_repo_path(message)
+  def sanitize_repo_path(project, message)
     return '' unless message.present?
-    message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]")
+    message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
   end
 end

app/models/namespace.rb

@@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base
   delegate :name, to: :owner, allow_nil: true, prefix: true
-  after_create :ensure_dir_exist
   after_update :move_dir, if: :path_changed?
+
+  # Save the storage paths before the projects are destroyed to use them on after destroy
+  before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths }
   after_destroy :rm_dir
   scope :root, -> { where('type IS NULL') }
@@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base
     owner_name
   end
-  def ensure_dir_exist
-    gitlab_shell.add_namespace(path)
-  end
-
-  def rm_dir
-    # Move namespace directory into trash.
-    # We will remove it later async
-    new_path = "#{path}+#{id}+deleted"
-
-    if gitlab_shell.mv_namespace(path, new_path)
-      message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
-      Gitlab::AppLogger.info message
-
-      # Remove namespace directroy async with delay so
-      # GitLab has time to remove all projects first
-      GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path)
-    end
-  end
-
   def move_dir
-    # Ensure old directory exists before moving it
-    gitlab_shell.add_namespace(path_was)
-
     if any_project_has_container_registry_tags?
       raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
     end
-    if gitlab_shell.mv_namespace(path_was, path)
-      Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
-
-      # If repositories moved successfully we need to
-      # send update instructions to users.
-      # However we cannot allow rollback since we moved namespace dir
-      # So we basically we mute exceptions in next actions
-      begin
-        send_update_instructions
-      rescue
-        # Returning false does not rollback after_* transaction but gives
-        # us information about failing some of tasks
-        false
+    # Move the namespace directory in all storages paths used by member projects
+    repository_storage_paths.each do |repository_storage_path|
+      # Ensure old directory exists before moving it
+      gitlab_shell.add_namespace(repository_storage_path, path_was)
+
+      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
+        # if we cannot move namespace directory we should rollback
+        # db changes in order to prevent out of sync between db and fs
+        raise Exception.new('namespace directory cannot be moved')
       end
-    else
-      # if we cannot move namespace directory we should rollback
-      # db changes in order to prevent out of sync between db and fs
-      raise Exception.new('namespace directory cannot be moved')
+    end
+
+    Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
+
+    # If repositories moved successfully we need to
+    # send update instructions to users.
+    # However we cannot allow rollback since we moved namespace dir
+    # So we basically we mute exceptions in next actions
+    begin
+      send_update_instructions
+    rescue
+      # Returning false does not rollback after_* transaction but gives
+      # us information about failing some of tasks
+      false
     end
   end
@@ -152,4 +138,33 @@ class Namespace < ActiveRecord::Base
   def find_fork_of(project)
     projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
   end
+
+  private
+
+  def repository_storage_paths
+    # We need to get the storage paths for all the projects, even the ones that are
+    # pending delete. Unscoping also get rids of the default order, which causes
+    # problems with SELECT DISTINCT.
+    Project.unscoped do
+      projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
+    end
+  end
+
+  def rm_dir
+    # Remove the namespace directory in all storages paths used by member projects
+    @old_repository_storage_paths.each do |repository_storage_path|
+      # Move namespace directory into trash.
+      # We will remove it later async
+      new_path = "#{path}+#{id}+deleted"
+
+      if gitlab_shell.mv_namespace(repository_storage_path, path, new_path)
+        message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
+        Gitlab::AppLogger.info message
+
+        # Remove namespace directroy async with delay so
+        # GitLab has time to remove all projects first
+        GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
+      end
+    end
+  end
 end

app/models/project.rb

@@ -26,6 +26,9 @@ class Project < ActiveRecord::Base
   default_value_for :container_registry_enabled, gitlab_config_features.container_registry
   default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
+  after_create :ensure_dir_exist
+  after_save :ensure_dir_exist, if: :namespace_id_changed?
+
   # set last_activity_at to the same as created_at
   after_create :set_last_activity_at
   def set_last_activity_at
@@ -165,6 +168,9 @@ class Project < ActiveRecord::Base
   validate :visibility_level_allowed_by_group
   validate :visibility_level_allowed_as_fork
   validate :check_wiki_path_conflict
+  validates :repository_storage,
+    presence: true,
+    inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
   add_authentication_token_field :runners_token
   before_save :ensure_runners_token
@@ -376,6 +382,10 @@ class Project < ActiveRecord::Base
     end
   end
+  def repository_storage_path
+    Gitlab.config.repositories.storages[repository_storage]
+  end
+
   def team
     @team ||= ProjectTeam.new(self)
   end
@@ -842,12 +852,12 @@ class Project < ActiveRecord::Base
       raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
     end
-    if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
+    if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
       # If repository moved successfully we need to send update instructions to users.
       # However we cannot allow rollback since we moved repository
       # So we basically we mute exceptions in next actions
       begin
-        gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
+        gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
         send_move_instructions(old_path_with_namespace)
         reset_events_cache
@@ -988,7 +998,7 @@ class Project < ActiveRecord::Base
   def create_repository
     # Forked import is handled asynchronously
     unless forked?
-      if gitlab_shell.add_repository(path_with_namespace)
+      if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
         repository.after_create
         true
       else
@@ -1140,4 +1150,8 @@ class Project < ActiveRecord::Base
     _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
     status.zero?
   end
+
+  def ensure_dir_exist
+    gitlab_shell.add_namespace(repository_storage_path, namespace.path)
+  end
 end

app/models/project_wiki.rb

@@ -159,7 +159,7 @@ class ProjectWiki
   private
   def init_repo(path_with_namespace)
-    gitlab_shell.add_repository(path_with_namespace)
+    gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace)
   end
   def commit_details(action, message = nil, title = nil)
@@ -173,7 +173,7 @@ class ProjectWiki
   end
   def path_to_repo
-    @path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git")
+    @path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git")
   end
   def update_project_activity

app/models/repository.rb

@@ -39,7 +39,7 @@ class Repository
   # Return absolute path to repository
   def path_to_repo
     @path_to_repo ||= File.expand_path(
-      File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git")
+      File.join(@project.repository_storage_path, path_with_namespace + ".git")
     )
   end

app/services/projects/destroy_service.rb

@@ -51,13 +51,13 @@ module Projects
       return true if params[:skip_repo] == true
       # There is a possibility project does not have repository or wiki
-      return true unless gitlab_shell.exists?(path + '.git')
+      return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
       new_path = removal_path(path)
-      if gitlab_shell.mv_repository(path, new_path)
+      if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
         log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
-        GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path)
+        GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path)
       else
         false
       end

app/services/projects/housekeeping_service.rb

@@ -24,7 +24,7 @@ module Projects
     def execute
       raise LeaseTaken unless try_obtain_lease
-      GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace)
+      GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace)
     ensure
       Gitlab::Metrics.measure(:reset_pushes_since_gc) do
         @project.update_column(:pushes_since_gc, 0)

app/services/projects/import_service.rb

@@ -42,7 +42,7 @@ module Projects
     def import_repository
       begin
-        gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
+        gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
       rescue Gitlab::Shell::Error => e
         raise Error,  "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
       end

app/services/projects/transfer_service.rb

@@ -50,12 +50,12 @@ module Projects
         project.send_move_instructions(old_path)
         # Move main repository
-        unless gitlab_shell.mv_repository(old_path, new_path)
+        unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path)
           raise TransferError.new('Cannot move project')
         end
         # Move wiki repo also if present
-        gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
+        gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki")
         # clear project cached events
         project.reset_events_cache

app/views/projects/imports/new.html.haml

@@ -10,7 +10,7 @@
     .panel-body
       %pre
         :preserve
-          #{sanitize_repo_path(@project.import_error)}
+          #{sanitize_repo_path(@project, @project.import_error)}
 = form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
   = render "shared/import_form", f: f

app/workers/post_receive.rb

@@ -4,10 +4,10 @@ class PostReceive
   sidekiq_options queue: :post_receive
   def perform(repo_path, identifier, changes)
-    if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s)
-      repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "")
+    if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
+      repo_path.gsub!(path[1].to_s, "")
     else
-      log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"")
+      log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
     end
     post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes)

app/workers/repository_fork_worker.rb

@@ -12,7 +12,7 @@ class RepositoryForkWorker
       return
     end
-    result = gitlab_shell.fork_repository(source_path, target_path)
+    result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path)
     unless result
       logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
       project.mark_import_as_failed('The project could not be forked.')

config/gitlab.teatro.yml

@@ -47,11 +47,13 @@ production: &base
   backup:
     path: "tmp/backups"   # Relative paths are relative to Rails.root (default: tmp/backups/)
+  repositories:
+    storages: # REPO PATHS MUST NOT BE A SYMLINK!!!
+      default: /apps/repositories/
+
   gitlab_shell:
     path: /apps/gitlab-shell/
-    # REPOS_PATH MUST NOT BE A SYMLINK!!!
-    repos_path: /apps/repositories/
     hooks_path: /apps/gitlab-shell/hooks/
     upload_pack: true

config/gitlab.yml.example

@@ -428,6 +428,13 @@ production: &base
   satellites:
     path: /home/git/gitlab-satellites/
+  ## Repositories settings
+  repositories:
+    # Paths where repositories can be stored. Give the canonicalized absolute pathname.
+    # NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
+    storages: # You must have at least a `default` storage path.
+      default: /home/git/repositories/
+
   ## Backup settings
   backup:
     path: "tmp/backups"   # Relative paths are relative to Rails.root (default: tmp/backups/)
@@ -452,9 +459,6 @@ production: &base
   ## GitLab Shell settings
   gitlab_shell:
     path: /home/git/gitlab-shell/
-
-    # REPOS_PATH MUST NOT BE A SYMLINK!!!
-    repos_path: /home/git/repositories/
     hooks_path: /home/git/gitlab-shell/hooks/
     # File that contains the secret key for verifying access for gitlab-shell.
@@ -528,11 +532,13 @@ test:
     # user: YOUR_USERNAME
   satellites:
     path: tmp/tests/gitlab-satellites/
+  repositories:
+    storages:
+      default: tmp/tests/repositories/
   backup:
     path: tmp/tests/backups
   gitlab_shell:
     path: tmp/tests/gitlab-shell/
-    repos_path: tmp/tests/repositories/
     hooks_path: tmp/tests/gitlab-shell/hooks/
   issues_tracker:
     redmine:

config/initializers/1_settings.rb

@@ -304,13 +304,20 @@ Settings.gitlab_shell['hooks_path']   ||= Settings.gitlab['user_home'] + '/gitla
 Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret')
 Settings.gitlab_shell['receive_pack']   = true if Settings.gitlab_shell['receive_pack'].nil?
 Settings.gitlab_shell['upload_pack']    = true if Settings.gitlab_shell['upload_pack'].nil?
-Settings.gitlab_shell['repos_path']   ||= Settings.gitlab['user_home'] + '/repositories/'
 Settings.gitlab_shell['ssh_host']     ||= Settings.gitlab.ssh_host
 Settings.gitlab_shell['ssh_port']     ||= 22
 Settings.gitlab_shell['ssh_user']     ||= Settings.gitlab.user
 Settings.gitlab_shell['owner_group']  ||= Settings.gitlab.user
 Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix)
+#
+# Repositories
+#
+Settings['repositories'] ||= Settingslogic.new({})
+Settings.repositories['storages'] ||= {}
+# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0
+Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/'
+
 #
 # Backup
 #

config/initializers/6_validations.rb

+def storage_name_valid?(name)
+  !!(name =~ /\A[a-zA-Z0-9\-_]+\z/)
+end
+
+def find_parent_path(name, path)
+  Gitlab.config.repositories.storages.detect do |n, p|
+    name != n && path.chomp('/').start_with?(p.chomp('/'))
+  end
+end
+
+def error(message)
+  raise "#{message}. Please fix this in your gitlab.yml before starting GitLab."
+end
+
+error('No repository storage path defined') if Gitlab.config.repositories.storages.empty?
+
+Gitlab.config.repositories.storages.each do |name, path|
+  error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name)
+
+  parent_name, _parent_path = find_parent_path(name, path)
+  if parent_name
+    error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages")
+  end
+end

db/migrate/20160608195742_add_repository_storage_to_projects.rb

+class AddRepositoryStorageToProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:projects, :repository_storage, :string, default: 'default')
+  end
+
+  def down
+    remove_column(:projects, :repository_storage)
+  end
+end

db/schema.rb

@@ -796,38 +796,39 @@ ActiveRecord::Schema.define(version: 20160620115026) do
     t.datetime "created_at"
     t.datetime "updated_at"
     t.integer  "creator_id"
-    t.boolean  "issues_enabled",                     default: true,  null: false
-    t.boolean  "merge_requests_enabled",             default: true,  null: false
-    t.boolean  "wiki_enabled",                       default: true,  null: false
+    t.boolean  "issues_enabled",                     default: true,      null: false
+    t.boolean  "merge_requests_enabled",             default: true,      null: false
+    t.boolean  "wiki_enabled",                       default: true,      null: false
     t.integer  "namespace_id"
-    t.boolean  "snippets_enabled",                   default: true,  null: false
+    t.boolean  "snippets_enabled",                   default: true,      null: false
     t.datetime "last_activity_at"
     t.string   "import_url"
-    t.integer  "visibility_level",                   default: 0,     null: false
-    t.boolean  "archived",                           default: false, null: false
+    t.integer  "visibility_level",                   default: 0,         null: false
+    t.boolean  "archived",                           default: false,     null: false
     t.string   "avatar"
     t.string   "import_status"
     t.float    "repository_size",                    default: 0.0
-    t.integer  "star_count",                         default: 0,     null: false
+    t.integer  "star_count",                         default: 0,         null: false
     t.string   "import_type"
     t.string   "import_source"
     t.integer  "commit_count",                       default: 0
     t.text     "import_error"
     t.integer  "ci_id"
-    t.boolean  "builds_enabled",                     default: true,  null: false
-    t.boolean  "shared_runners_enabled",             default: true,  null: false
+    t.boolean  "builds_enabled",                     default: true,      null: false
+    t.boolean  "shared_runners_enabled",             default: true,      null: false
     t.string   "runners_token"
     t.string   "build_coverage_regex"
-    t.boolean  "build_allow_git_fetch",              default: true,  null: false
-    t.integer  "build_timeout",                      default: 3600,  null: false
+    t.boolean  "build_allow_git_fetch",              default: true,      null: false
+    t.integer  "build_timeout",                      default: 3600,      null: false
     t.boolean  "pending_delete",                     default: false
-    t.boolean  "public_builds",                      default: true,  null: false
+    t.boolean  "public_builds",                      default: true,      null: false
     t.integer  "pushes_since_gc",                    default: 0
     t.boolean  "last_repository_check_failed"
     t.datetime "last_repository_check_at"
     t.boolean  "container_registry_enabled"
-    t.boolean  "only_allow_merge_if_build_succeeds", default: false, null: false
+    t.boolean  "only_allow_merge_if_build_succeeds", default: false,     null: false
     t.boolean  "has_external_issue_tracker"
+    t.string   "repository_storage",                 default: "default", null: false
   end
   add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree