Extend API for importing a project export with overwrite support

app/models/group.rb

@@ -286,6 +286,10 @@ class Group < Namespace
     false
   end
+  def refresh_project_authorizations
+    refresh_members_authorized_projects(blocking: false)
+  end
+
   private
   def update_two_factor_requirement

app/models/namespace.rb

@@ -252,6 +252,10 @@ class Namespace < ActiveRecord::Base
     []
   end
+  def refresh_project_authorizations
+    owner.refresh_authorized_projects
+  end
+
   private
   def path_or_parent_changed?

app/models/project.rb

@@ -1472,7 +1472,9 @@ class Project < ActiveRecord::Base
   end
   def rename_repo_notify!
-    send_move_instructions(full_path_was)
+    # When we import a project overwriting the original project, there
+    # is a move operation. In that case we don't want to send the instructions.
+    send_move_instructions(full_path_was) unless started?
     expires_full_path_cache
     self.old_path_with_namespace = full_path_was

app/services/projects/base_move_relations_service.rb

+module Projects
+  class BaseMoveRelationsService < BaseService
+    attr_reader :source_project
+    def execute(source_project, remove_remaining_elements: true)
+      return if source_project.blank?
+
+      @source_project = source_project
+
+      true
+    end
+
+    private
+
+    def prepare_relation(relation, id_param = :id)
+      if Gitlab::Database.postgresql?
+        relation
+      else
+        relation.model.where("#{id_param}": relation.pluck(id_param))
+      end
+    end
+  end
+end

app/services/projects/destroy_service.rb

@@ -46,6 +46,20 @@ module Projects
       raise
     end
+    def attempt_repositories_rollback
+      return unless @project
+
+      flush_caches(@project)
+
+      unless mv_repository(removal_path(repo_path), repo_path)
+        raise_error('Failed to restore project repository. Please contact the administrator.')
+      end
+
+      unless mv_repository(removal_path(wiki_path), wiki_path)
+        raise_error('Failed to restore wiki repository. Please contact the administrator.')
+      end
+    end
+
     private
     def repo_path
@@ -70,12 +84,9 @@ module Projects
       # Skip repository removal. We use this flag when remove user or group
       return true if params[:skip_repo] == true
-      # There is a possibility project does not have repository or wiki
-      return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
-
       new_path = removal_path(path)
-      if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
+      if mv_repository(path, new_path)
         log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
         project.run_after_commit do
@@ -87,6 +98,13 @@ module Projects
       end
     end
+    def mv_repository(from_path, to_path)
+      # There is a possibility project does not have repository or wiki
+      return true unless gitlab_shell.exists?(project.repository_storage_path, from_path + '.git')
+
+      gitlab_shell.mv_repository(project.repository_storage_path, from_path, to_path)
+    end
+
     def attempt_rollback(project, message)
       return unless project

app/services/projects/gitlab_projects_import_service.rb

@@ -15,9 +15,18 @@ module Projects
       file = params.delete(:file)
       FileUtils.copy_entry(file.path, import_upload_path)
+      @overwrite = params.delete(:overwrite)
+      data = {}
+      data[:override_params] = @override_params if @override_params
+
+      if overwrite_project?
+        data[:original_path] = params[:path]
+        params[:path] += "-#{tmp_filename}"
+      end
+
       params[:import_type] = 'gitlab_project'
       params[:import_source] = import_upload_path
-      params[:import_data] = { data: { override_params: @override_params } } if @override_params
+      params[:import_data] = { data: data } if data.present?
       ::Projects::CreateService.new(current_user, params).execute
     end
@@ -31,5 +40,17 @@ module Projects
     def tmp_filename
       SecureRandom.hex
     end
+
+    def overwrite_project?
+      @overwrite && project_with_same_full_path?
+    end
+
+    def project_with_same_full_path?
+      Project.find_by_full_path("#{current_namespace.full_path}/#{params[:path]}").present?
+    end
+
+    def current_namespace
+      @current_namespace ||= Namespace.find_by(id: params[:namespace_id])
+    end
   end
 end

app/services/projects/move_access_service.rb

+module Projects
+  class MoveAccessService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      @project.with_transaction_returning_status do
+        if @project.namespace != source_project.namespace
+          @project.run_after_commit do
+            source_project.namespace.refresh_project_authorizations
+            self.namespace.refresh_project_authorizations
+          end
+        end
+
+        ::Projects::MoveProjectMembersService.new(@project, @current_user)
+          .execute(source_project, remove_remaining_elements: remove_remaining_elements)
+        ::Projects::MoveProjectGroupLinksService.new(@project, @current_user)
+          .execute(source_project, remove_remaining_elements: remove_remaining_elements)
+        ::Projects::MoveProjectAuthorizationsService.new(@project, @current_user)
+          .execute(source_project, remove_remaining_elements: remove_remaining_elements)
+
+        success
+      end
+    end
+  end
+end

app/services/projects/move_deploy_keys_projects_service.rb

+module Projects
+  class MoveDeployKeysProjectsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_deploy_keys_projects
+        remove_remaining_deploy_keys_projects if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_deploy_keys_projects
+      prepare_relation(non_existent_deploy_keys_projects)
+        .update_all(project_id: @project.id)
+    end
+
+    def non_existent_deploy_keys_projects
+      source_project.deploy_keys_projects
+                    .joins(:deploy_key)
+                    .where.not(keys: { fingerprint: @project.deploy_keys.select(:fingerprint) })
+    end
+
+    def remove_remaining_deploy_keys_projects
+      source_project.deploy_keys_projects.destroy_all
+    end
+  end
+end

app/services/projects/move_forks_service.rb

+module Projects
+  class MoveForksService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super && source_project.fork_network
+
+      Project.transaction(requires_new: true) do
+        move_forked_project_links
+        move_fork_network_members
+        update_root_project
+        refresh_forks_count
+
+        success
+      end
+    end
+
+    private
+
+    def move_forked_project_links
+      # Update ancestor
+      ForkedProjectLink.where(forked_to_project: source_project)
+                       .update_all(forked_to_project_id: @project.id)
+
+      # Update the descendants
+      ForkedProjectLink.where(forked_from_project: source_project)
+                       .update_all(forked_from_project_id: @project.id)
+    end
+
+    def move_fork_network_members
+      ForkNetworkMember.where(project: source_project).update_all(project_id: @project.id)
+      ForkNetworkMember.where(forked_from_project: source_project).update_all(forked_from_project_id: @project.id)
+    end
+
+    def update_root_project
+      # Update root network project
+      ForkNetwork.where(root_project: source_project).update_all(root_project_id: @project.id)
+    end
+
+    def refresh_forks_count
+      Projects::ForksCountService.new(@project).refresh_cache
+    end
+  end
+end

app/services/projects/move_lfs_objects_projects_service.rb

+module Projects
+  class MoveLfsObjectsProjectsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_lfs_objects_projects
+        remove_remaining_lfs_objects_project if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_lfs_objects_projects
+      prepare_relation(non_existent_lfs_objects_projects)
+        .update_all(project_id: @project.lfs_storage_project.id)
+    end
+
+    def remove_remaining_lfs_objects_project
+      source_project.lfs_objects_projects.destroy_all
+    end
+
+    def non_existent_lfs_objects_projects
+      source_project.lfs_objects_projects.where.not(lfs_object: @project.lfs_objects)
+    end
+  end
+end

app/services/projects/move_notification_settings_service.rb

+module Projects
+  class MoveNotificationSettingsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_notification_settings
+        remove_remaining_notification_settings if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_notification_settings
+      prepare_relation(non_existent_notifications)
+        .update_all(source_id: @project.id)
+    end
+
+    # Remove remaining notification settings from source_project
+    def remove_remaining_notification_settings
+      source_project.notification_settings.destroy_all
+    end
+
+    # Get users of current notification_settings
+    def users_in_target_project
+      @project.notification_settings.select(:user_id)
+    end
+
+    # Look for notification_settings in source_project that are not in the target project
+    def non_existent_notifications
+      source_project.notification_settings
+        .select(:id)
+        .where.not(user_id: users_in_target_project)
+    end
+  end
+end

app/services/projects/move_project_authorizations_service.rb

+# NOTE: This service cannot be used directly because it is part of a
+# a bigger process. Instead, use the service MoveAccessService which moves
+# project memberships, project group links, authorizations and refreshes
+# the authorizations if neccessary
+module Projects
+  class MoveProjectAuthorizationsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_project_authorizations
+
+        remove_remaining_authorizations if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_project_authorizations
+      prepare_relation(non_existent_authorization, :user_id)
+        .update_all(project_id: @project.id)
+    end
+
+    def remove_remaining_authorizations
+      # I think because the Project Authorization table does not have a primary key
+      # it brings a lot a problems/bugs. First, Rails raises PG::SyntaxException if we use
+      # destroy_all instead of delete_all.
+      source_project.project_authorizations.delete_all(:delete_all)
+    end
+
+    # Look for authorizations in source_project that are not in the target project
+    def non_existent_authorization
+      source_project.project_authorizations
+                    .select(:user_id)
+                    .where.not(user: @project.authorized_users)
+    end
+  end
+end

app/services/projects/move_project_group_links_service.rb

+# NOTE: This service cannot be used directly because it is part of a
+# a bigger process. Instead, use the service MoveAccessService which moves
+# project memberships, project group links, authorizations and refreshes
+# the authorizations if neccessary
+module Projects
+  class MoveProjectGroupLinksService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_group_links
+        remove_remaining_project_group_links if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_group_links
+      prepare_relation(non_existent_group_links)
+        .update_all(project_id: @project.id)
+    end
+
+    # Remove remaining project group links from source_project
+    def remove_remaining_project_group_links
+      source_project.reload.project_group_links.destroy_all
+    end
+
+    def group_links_in_target_project
+      @project.project_group_links.select(:group_id)
+    end
+
+    # Look for groups in source_project that are not in the target project
+    def non_existent_group_links
+      source_project.project_group_links
+                    .where.not(group_id: group_links_in_target_project)
+    end
+  end
+end

app/services/projects/move_project_members_service.rb

+# NOTE: This service cannot be used directly because it is part of a
+# a bigger process. Instead, use the service MoveAccessService which moves
+# project memberships, project group links, authorizations and refreshes
+# the authorizations if neccessary
+module Projects
+  class MoveProjectMembersService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_project_members
+        remove_remaining_members if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_project_members
+      prepare_relation(non_existent_members).update_all(source_id: @project.id)
+    end
+
+    def remove_remaining_members
+      # Remove remaining members and authorizations from source_project
+      source_project.project_members.destroy_all
+    end
+
+    def project_members_in_target_project
+      @project.project_members.select(:user_id)
+    end
+
+    # Look for members in source_project that are not in the target project
+    def non_existent_members
+      source_project.members
+                    .select(:id)
+                    .where.not(user_id: @project.project_members.select(:user_id))
+    end
+  end
+end

app/services/projects/move_users_star_projects_service.rb

+module Projects
+  class MoveUsersStarProjectsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      user_stars = source_project.users_star_projects
+
+      return unless user_stars.any?
+
+      Project.transaction(requires_new: true) do
+        user_stars.update_all(project_id: @project.id)
+
+        Project.reset_counters @project.id, :users_star_projects
+        Project.reset_counters source_project.id, :users_star_projects
+
+        success
+      end
+    end
+  end
+end

app/services/projects/overwrite_project_service.rb

+module Projects
+  class OverwriteProjectService < BaseService
+    def execute(source_project)
+      return unless source_project && source_project.namespace == @project.namespace
+
+      Project.transaction do
+        move_before_destroy_relationships(source_project)
+        destroy_old_project(source_project)
+        rename_project(source_project.name, source_project.path)
+
+        @project
+      end
+    # Projects::DestroyService can raise Exceptions, but we don't want
+    # to pass that kind of exception to the caller. Instead, we change it
+    # for a StandardError exception
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      attempt_restore_repositories(source_project)
+
+      if e.class == Exception
+        raise StandardError, e.message
+      else
+        raise
+      end
+    end
+
+    private
+
+    def move_before_destroy_relationships(source_project)
+      options = { remove_remaining_elements: false }
+
+      ::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveForksService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, options)
+      add_source_project_to_fork_network(source_project)
+    end
+
+    def destroy_old_project(source_project)
+      # Delete previous project (synchronously) and unlink relations
+      ::Projects::DestroyService.new(source_project, @current_user).execute
+    end
+
+    def rename_project(name, path)
+      # Update de project's name and path to the original name/path
+      ::Projects::UpdateService.new(@project,
+                                    @current_user,
+                                    { name: name, path: path })
+                               .execute
+    end
+
+    def attempt_restore_repositories(project)
+      ::Projects::DestroyService.new(project, @current_user).attempt_repositories_rollback
+    end
+
+    def add_source_project_to_fork_network(source_project)
+      return unless @project.fork_network
+
+      # Because he have moved all references in the fork network from the source_project
+      # we won't be able to query the database (only through its cached data),
+      # for its former relationships. That's why we're adding it to the network
+      # as a fork of the target project
+      ForkNetworkMember.create!(fork_network: @project.fork_network,
+                                project: source_project,
+                                forked_from_project: @project)
+    end
+  end
+end

changelogs/unreleased/fj-41900-import-endpoint-with-overwrite-support.yml

+---
+title: Extend API for importing a project export with overwrite support
+merge_request: 17883
+author:
+type: added

doc/api/project_import_export.md

@@ -111,6 +111,7 @@ POST /projects/import
 | `namespace` | integer/string | no | The ID or path of the namespace that the project will be imported to. Defaults to the current user's namespace |
 | `file` | string | yes | The file to be uploaded |
 | `path` | string | yes | Name and path for new project |
+| `overwrite` | boolean | no | If there is a project with the same path the import will overwrite it. Default to false |
 | `override_params` | Hash | no | Supports all fields defined in the [Project API](projects.md)] |
 The override params passed will take precendence over all values defined inside the export file.

lib/api/project_import.rb

@@ -26,6 +26,7 @@ module API
         requires :path, type: String, desc: 'The new project path and name'
         requires :file, type: File, desc: 'The project export file to be imported'
         optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace."
+        optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it'
         optional :override_params,
                  type: Hash,
                  desc: 'New project params to override values in the export' do
@@ -50,7 +51,8 @@ module API
         project_params = {
             path: import_params[:path],
             namespace_id: namespace.id,
-            file: import_params[:file]['tempfile']
+            file: import_params[:file]['tempfile'],
+            overwrite: import_params[:overwrite]
         }
         override_params = import_params.delete(:override_params)

lib/gitlab/import_export/importer.rb

 module Gitlab
   module ImportExport
     class Importer
+      include Gitlab::Allowable
+      include Gitlab::Utils::StrongMemoize
+
       def self.imports_repository?
         true
       end
@@ -13,12 +16,14 @@ module Gitlab
       end
       def execute
-        if import_file && check_version! && restorers.all?(&:restore)
+        if import_file && check_version! && restorers.all?(&:restore) && overwrite_project
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
         end
-
+      rescue => e
+        raise Projects::ImportService::Error.new(e.message)
+      ensure
         remove_import_file
       end
@@ -26,7 +31,7 @@ module Gitlab
       def restorers
         [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
-         uploads_restorer, lfs_restorer]
+         uploads_restorer, lfs_restorer, statistics_restorer]
       end
       def import_file
@@ -69,6 +74,10 @@ module Gitlab
         Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
       end
+      def statistics_restorer
+        Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared)
+      end
+
       def path_with_namespace
         File.join(@project.namespace.full_path, @project.path)
       end
@@ -84,6 +93,33 @@ module Gitlab
       def remove_import_file
         FileUtils.rm_rf(@archive_file)
       end
+
+      def overwrite_project
+        project = project_tree.restored_project
+
+        return unless can?(@current_user, :admin_namespace, project.namespace)
+
+        if overwrite_project?
+          ::Projects::OverwriteProjectService.new(project, @current_user)
+                                             .execute(project_to_overwrite)
+        end
+
+        true
+      end
+
+      def original_path
+        @project.import_data&.data&.fetch('original_path', nil)
+      end
+
+      def overwrite_project?
+        original_path.present? && project_to_overwrite.present?
+      end
+
+      def project_to_overwrite
+        strong_memoize(:project_to_overwrite) do
+          Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}")
+        end
+      end
     end
   end
 end