Gitlab::Shell works on shard name, not path

Direct disk access is done through Gitaly now, so the legacy path was
deprecated. This path was used in Gitlab::Shell however. This required
the refactoring in this commit.

Added is the removal of direct path access on the project model, as that
lookup wasn't needed anymore is most cases.

Closes https://gitlab.com/gitlab-org/gitaly/issues/1111

app/helpers/projects_helper.rb

@@ -400,7 +400,8 @@ module ProjectsHelper
     exports_path = File.join(Settings.shared['path'], 'tmp/project_exports')
     filtered_message = message.strip.gsub(exports_path, "[REPO EXPORT PATH]")
-    filtered_message.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
+    disk_path = Gitlab.config.repositories.storages[project.repository_storage].legacy_disk_path
+    filtered_message.gsub(disk_path.chomp('/'), "[REPOS PATH]")
   end
   def project_child_container_class(view_path)

app/models/concerns/storage/legacy_namespace.rb

@@ -45,25 +45,25 @@ module Storage
     # Hooks
-    # Save the storage paths before the projects are destroyed to use them on after destroy
+    # Save the storages before the projects are destroyed to use them on after destroy
     def prepare_for_destroy
-      old_repository_storage_paths
+      old_repository_storages
     end
     private
     def move_repositories
-      # Move the namespace directory in all storage paths used by member projects
-      repository_storage_paths.each do |repository_storage_path|
+      # Move the namespace directory in all storages used by member projects
+      repository_storages.each do |repository_storage|
         # Ensure old directory exists before moving it
-        gitlab_shell.add_namespace(repository_storage_path, full_path_was)
+        gitlab_shell.add_namespace(repository_storage, full_path_was)
         # Ensure new directory exists before moving it (if there's a parent)
-        gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent
-        unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
-          Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
+        gitlab_shell.add_namespace(repository_storage, parent.full_path) if parent
+        unless gitlab_shell.mv_namespace(repository_storage, full_path_was, full_path)
+          Rails.logger.error "Exception moving path #{repository_storage} from #{full_path_was} to #{full_path}"
           # if we cannot move namespace directory we should rollback
           # db changes in order to prevent out of sync between db and fs
@@ -72,33 +72,33 @@ module Storage
       end
     end
-    def old_repository_storage_paths
-      @old_repository_storage_paths ||= repository_storage_paths
+    def old_repository_storages
+      @old_repository_storage_paths ||= repository_storages
     end
-    def repository_storage_paths
+    def repository_storages
       # We need to get the storage paths for all the projects, even the ones that are
       # pending delete. Unscoping also get rids of the default order, which causes
       # problems with SELECT DISTINCT.
       Project.unscoped do
-        all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
+        all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage)
       end
     end
     def rm_dir
       # Remove the namespace directory in all storages paths used by member projects
-      old_repository_storage_paths.each do |repository_storage_path|
+      old_repository_storages.each do |repository_storage|
         # Move namespace directory into trash.
         # We will remove it later async
         new_path = "#{full_path}+#{id}+deleted"
-        if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path)
+        if gitlab_shell.mv_namespace(repository_storage, full_path, new_path)
           Gitlab::AppLogger.info %Q(Namespace directory "#{full_path}" moved to "#{new_path}")
           # Remove namespace directroy async with delay so
           # GitLab has time to remove all projects first
           run_after_commit do
-            GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
+            GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage, new_path)
           end
         end
       end

app/models/project.rb

@@ -512,10 +512,6 @@ class Project < ActiveRecord::Base
     repository.empty?
   end
-  def repository_storage_path
-    Gitlab.config.repositories.storages[repository_storage]&.legacy_disk_path
-  end
-
   def team
     @team ||= ProjectTeam.new(self)
   end
@@ -1106,7 +1102,7 @@ class Project < ActiveRecord::Base
   # Check if repository already exists on disk
   def check_repository_path_availability
     return true if skip_disk_validation
-    return false unless repository_storage_path
+    return false unless repository_storage
     expires_full_path_cache # we need to clear cache to validate renames correctly
@@ -1907,14 +1903,14 @@ class Project < ActiveRecord::Base
   def check_repository_absence!
     return if skip_disk_validation
-    if repository_storage_path.blank? || repository_with_same_path_already_exists?
+    if repository_storage.blank? || repository_with_same_path_already_exists?
       errors.add(:base, 'There is already a repository with that name on disk')
       throw :abort
     end
   end
   def repository_with_same_path_already_exists?
-    gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git")
+    gitlab_shell.exists?(repository_storage, "#{disk_path}.git")
   end
   # set last_activity_at to the same as created_at

app/models/project_wiki.rb

@@ -21,7 +21,7 @@ class ProjectWiki
   end
   delegate :empty?, to: :pages
-  delegate :repository_storage_path, :hashed_storage?, to: :project
+  delegate :repository_storage, :hashed_storage?, to: :project
   def path
     @project.path + '.wiki'

app/models/repository.rb

@@ -84,9 +84,14 @@ class Repository
   # Return absolute path to repository
   def path_to_repo
-    @path_to_repo ||= File.expand_path(
-      File.join(repository_storage_path, disk_path + '.git')
-    )
+    @path_to_repo ||=
+      begin
+        storage = Gitlab.config.repositories.storages[@project.repository_storage]
+
+        File.expand_path(
+          File.join(storage.legacy_disk_path, disk_path + '.git')
+        )
+      end
   end
   def inspect
@@ -915,10 +920,6 @@ class Repository
     raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
   end
-  def repository_storage_path
-    @project.repository_storage_path
-  end
-
   def rebase(user, merge_request)
     raw.rebase(user, merge_request.id, branch: merge_request.source_branch,
                                        branch_sha: merge_request.source_branch_sha,

app/models/storage/hashed_project.rb

 module Storage
   class HashedProject
     attr_accessor :project
-    delegate :gitlab_shell, :repository_storage_path, to: :project
+    delegate :gitlab_shell, :repository_storage, to: :project
     ROOT_PATH_PREFIX = '@hashed'.freeze
@@ -24,7 +24,7 @@ module Storage
     end
     def ensure_storage_path_exists
-      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+      gitlab_shell.add_namespace(repository_storage, base_dir)
     end
     def rename_repo

app/models/storage/legacy_project.rb

 module Storage
   class LegacyProject
     attr_accessor :project
-    delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project
+    delegate :namespace, :gitlab_shell, :repository_storage, to: :project
     def initialize(project)
       @project = project
@@ -24,18 +24,18 @@ module Storage
     def ensure_storage_path_exists
       return unless namespace
-      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+      gitlab_shell.add_namespace(repository_storage, base_dir)
     end
     def rename_repo
       new_full_path = project.build_full_path
-      if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path)
+      if gitlab_shell.mv_repository(repository_storage, project.full_path_was, new_full_path)
         # If repository moved successfully we need to send update instructions to users.
         # However we cannot allow rollback since we moved repository
         # So we basically we mute exceptions in next actions
         begin
-          gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
+          gitlab_shell.mv_repository(repository_storage, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
           return true
         rescue => e
           Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}"

app/services/projects/destroy_service.rb

@@ -91,7 +91,7 @@ module Projects
         project.run_after_commit do
           # self is now project
-          GitlabShellWorker.perform_in(5.minutes, :remove_repository, self.repository_storage_path, new_path)
+          GitlabShellWorker.perform_in(5.minutes, :remove_repository, self.repository_storage, new_path)
         end
       else
         false
@@ -100,9 +100,9 @@ module Projects
     def mv_repository(from_path, to_path)
       # There is a possibility project does not have repository or wiki
-      return true unless gitlab_shell.exists?(project.repository_storage_path, from_path + '.git')
-      gitlab_shell.mv_repository(project.repository_storage_path, from_path, to_path)
+      return true unless gitlab_shell.exists?(project.repository_storage, from_path + '.git')
+      gitlab_shell.mv_repository(project.repository_storage, from_path, to_path)
     end
     def attempt_rollback(project, message)

app/services/projects/hashed_storage/migrate_repository_service.rb

@@ -47,8 +47,8 @@ module Projects
       private
       def move_repository(from_name, to_name)
-        from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
-        to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
+        from_exists = gitlab_shell.exists?(project.repository_storage, "#{from_name}.git")
+        to_exists = gitlab_shell.exists?(project.repository_storage, "#{to_name}.git")
         # If we don't find the repository on either original or target we should log that as it could be an issue if the
         # project was not originally empty.
@@ -60,7 +60,7 @@ module Projects
           return true
         end
-        gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+        gitlab_shell.mv_repository(project.repository_storage, from_name, to_name)
       end
       def rollback_folder_move

app/services/projects/transfer_service.rb

@@ -127,7 +127,7 @@ module Projects
     end
     def move_repo_folder(from_name, to_name)
-      gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+      gitlab_shell.mv_repository(project.repository_storage, from_name, to_name)
     end
     def execute_system_hooks

app/workers/repository_fork_worker.rb

@@ -13,7 +13,9 @@ class RepositoryForkWorker
     # See https://gitlab.com/gitlab-org/gitaly/issues/1110
     if args.empty?
       source_project = target_project.forked_from_project
-      return target_project.mark_import_as_failed('Source project cannot be found.') unless source_project
+      unless source_project
+        return target_project.mark_import_as_failed('Source project cannot be found.')
+      end
       fork_repository(target_project, source_project.repository_storage, source_project.disk_path)
     else

db/migrate/20161220141214_remove_dot_git_from_group_names.rb

@@ -59,17 +59,17 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration
   end
   def move_namespace(group_id, path_was, path)
-    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
-      Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
+    repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
+      row['repository_storage']
     end.compact
     # Move the namespace directory in all storages paths used by member projects
-    repository_storage_paths.each do |repository_storage_path|
+    repository_storages.each do |repository_storage|
       # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, path_was)
-      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
-        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+      gitlab_shell.add_namespace(repository_storage, path_was)
+      unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
+        Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
         # if we cannot move namespace directory we should rollback
         # db changes in order to prevent out of sync between db and fs

db/migrate/20161226122833_remove_dot_git_from_usernames.rb

@@ -53,8 +53,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
     select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(path)}'").present?
   end
-  def path_exists?(path, repository_storage_path)
-    repository_storage_path && gitlab_shell.exists?(repository_storage_path, path)
+  def path_exists?(shard, repository_storage_path)
+    repository_storage_path && gitlab_shell.exists?(shard, repository_storage_path)
   end
   # Accepts invalid path like test.git and returns test_git or
@@ -70,8 +70,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
   def check_routes(base, counter, path)
     route_exists = route_exists?(path)
-    Gitlab.config.repositories.storages.each_value do |storage|
-      if route_exists || path_exists?(path, storage.legacy_disk_path)
+    Gitlab.config.repositories.storages.each do |shard, storage|
+      if route_exists || path_exists?(shard, storage.legacy_disk_path)
         counter += 1
         path = "#{base}#{counter}"
@@ -83,17 +83,17 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
   end
   def move_namespace(namespace_id, path_was, path)
-    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
-      Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
+    repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
+      row['repository_storage']
     end.compact
-    # Move the namespace directory in all storages paths used by member projects
-    repository_storage_paths.each do |repository_storage_path|
+    # Move the namespace directory in all storages used by member projects
+    repository_storages.each do |repository_storage|
       # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, path_was)
-      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
-        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+      gitlab_shell.add_namespace(repository_storage, path_was)
+      unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
+        Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
         # if we cannot move namespace directory we should rollback
         # db changes in order to prevent out of sync between db and fs

lib/gitlab/bare_repository_import/importer.rb

@@ -75,10 +75,11 @@ module Gitlab
       end
       def mv_repo(project)
-        FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git'))
+        storage_path = storage_path_for_shard(project.repository_storage)
+        FileUtils.mv(repo_path, project.repository.path_to_repo)
         if bare_repo.wiki_exists?
-          FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git'))
+          FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git'))
         end
         true
@@ -88,6 +89,10 @@ module Gitlab
         false
       end
+      def storage_path_for_shard(shard)
+        Gitlab.config.repositories.storages[shard].legacy_disk_path
+      end
+
       def find_or_create_groups
         return nil unless group_path.present?

lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb

@@ -62,21 +62,20 @@ module Gitlab
           end
           def move_repositories(namespace, old_full_path, new_full_path)
-            repo_paths_for_namespace(namespace).each do |repository_storage_path|
+            repo_shards_for_namespace(namespace).each do |repository_storage|
               # Ensure old directory exists before moving it
-              gitlab_shell.add_namespace(repository_storage_path, old_full_path)
-              unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
-                message = "Exception moving path #{repository_storage_path} \
-                           from #{old_full_path} to #{new_full_path}"
+              gitlab_shell.add_namespace(repository_storage, old_full_path)
+              unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path)
+                message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}"
                 Rails.logger.error message
               end
             end
           end
-          def repo_paths_for_namespace(namespace)
+          def repo_shards_for_namespace(namespace)
             projects_for_namespace(namespace).distinct.select(:repository_storage)
-              .map(&:repository_storage_path)
+              .map(&:repository_storage)
           end
           def projects_for_namespace(namespace)

lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb

@@ -51,7 +51,7 @@ module Gitlab
           end
           def move_repository(project, old_path, new_path)
-            unless gitlab_shell.mv_repository(project.repository_storage_path,
+            unless gitlab_shell.mv_repository(project.repository_storage,
                                               old_path,
                                               new_path)
               Rails.logger.error "Error moving #{old_path} to #{new_path}"

lib/gitlab/shell.rb

@@ -65,11 +65,11 @@ module Gitlab
     # Init new repository
     #
-    # storage - project's storage name
+    # storage - the shard key
     # name - project disk path
     #
     # Ex.
-    #   create_repository("/path/to/storage", "gitlab/gitlab-ci")
+    #   create_repository("default", "gitlab/gitlab-ci")
     #
     def create_repository(storage, name)
       relative_path = name.dup
@@ -291,13 +291,13 @@ module Gitlab
     # Add empty directory for storing repositories
     #
     # Ex.
-    #   add_namespace("/path/to/storage", "gitlab")
+    #   add_namespace("default", "gitlab")
     #
     def add_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:add_namespace,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).add(name)
+          Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
         else
           path = full_path(storage, name)
           FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
@@ -313,13 +313,13 @@ module Gitlab
     # Every repository inside this directory will be removed too
     #
     # Ex.
-    #   rm_namespace("/path/to/storage", "gitlab")
+    #   rm_namespace("default", "gitlab")
     #
     def rm_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:remove_namespace,
                                status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).remove(name)
+          Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
         else
           FileUtils.rm_r(full_path(storage, name), force: true)
         end
@@ -338,7 +338,8 @@ module Gitlab
       Gitlab::GitalyClient.migrate(:rename_namespace,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).rename(old_name, new_name)
+          Gitlab::GitalyClient::NamespaceService.new(storage)
+            .rename(old_name, new_name)
         else
           break false if exists?(storage, new_name) || !exists?(storage, old_name)
@@ -374,7 +375,8 @@ module Gitlab
       Gitlab::GitalyClient.migrate(:namespace_exists,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).exists?(dir_name)
+          Gitlab::GitalyClient::NamespaceService.new(storage)
+            .exists?(dir_name)
         else
           File.exist?(full_path(storage, dir_name))
         end
@@ -398,7 +400,7 @@ module Gitlab
     def full_path(storage, dir_name)
       raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
-      File.join(storage, dir_name)
+      File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name)
     end
     def gitlab_shell_projects_path
@@ -475,14 +477,6 @@ module Gitlab
       Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
     end
-    def gitaly_namespace_client(storage_path)
-      storage, _value = Gitlab.config.repositories.storages.find do |storage, value|
-        value.legacy_disk_path == storage_path
-      end
-
-      Gitlab::GitalyClient::NamespaceService.new(storage)
-    end
-
     def git_timeout
       Gitlab.config.gitlab_shell.git_timeout
     end

lib/tasks/gitlab/check.rake

@@ -427,10 +427,7 @@ namespace :gitlab do
       user = User.find_by(username: username)
       if user
         repo_dirs = user.authorized_projects.map do |p|
-          File.join(
-            p.repository_storage_path,
-            "#{p.disk_path}.git"
-          )
+          p.repository.path_to_repo
         end
         repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }

lib/tasks/gitlab/list_repos.rake

@@ -10,9 +10,8 @@ namespace :gitlab do
     end
     scope.find_each do |project|
-      base = File.join(project.repository_storage_path, project.disk_path)
-      puts base + '.git'
-      puts base + '.wiki.git'
+      puts project.repository.path_to_repo
+      puts project.wiki.repository.path_to_repo
     end
   end
 end

spec/controllers/profiles_controller_spec.rb

@@ -125,7 +125,7 @@ describe ProfilesController, :request_store do
         user.reload
         expect(response.status).to eq(302)
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{new_username}/#{project.path}.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, "#{new_username}/#{project.path}.git")).to be_truthy
       end
     end
@@ -143,7 +143,7 @@ describe ProfilesController, :request_store do
         user.reload
         expect(response.status).to eq(302)
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be_truthy
         expect(before_disk_path).to eq(project.disk_path)
       end
     end