Merge branch '48778-remove-old-storage-logic-from-import-export' into 'master'

Resolve "Remove old storage logic from Import/Export"

Closes #51144, #50109, #48778, and #51079

See merge request gitlab-org/gitlab-ce!21409

app/controllers/projects_controller.rb

@@ -191,10 +191,8 @@ class ProjectsController < Projects::ApplicationController
   end
   def download_export
-    if export_project_object_storage?
-      send_upload(@project.import_export_upload.export_file)
-    elsif export_project_path
-      send_file export_project_path, disposition: 'attachment'
+    if @project.export_file_exists?
+      send_upload(@project.export_file)
     else
       redirect_to(
         edit_project_path(@project, anchor: 'js-export-project'),
@@ -425,12 +423,4 @@ class ProjectsController < Projects::ApplicationController
   def whitelist_query_limiting
     Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
   end
-
-  def export_project_path
-    @export_project_path ||= @project.export_project_path
-  end
-
-  def export_project_object_storage?
-    @project.export_project_object_exists?
-  end
 end

app/models/concerns/storage/legacy_namespace.rb

@@ -25,8 +25,6 @@ module Storage
         Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
       end
-      remove_exports!
-
       # If repositories moved successfully we need to
       # send update instructions to users.
       # However we cannot allow rollback since we moved namespace dir
@@ -101,8 +99,6 @@ module Storage
           end
         end
       end
-
-      remove_exports!
     end
     def remove_legacy_exports!

app/models/namespace.rb

@@ -253,18 +253,6 @@ class Namespace < ActiveRecord::Base
     end
   end
-  # Exports belonging to projects with legacy storage are placed in a common
-  # subdirectory of the namespace, so a simple `rm -rf` is sufficient to remove
-  # them.
-  #
-  # Exports of projects using hashed storage are placed in a location defined
-  # only by the project ID, so each must be removed individually.
-  def remove_exports!
-    remove_legacy_exports!
-
-    all_projects.with_storage_feature(:repository).find_each(&:remove_exports)
-  end
-
   def refresh_project_authorizations
     owner.refresh_authorized_projects
   end

app/models/project.rb

@@ -1735,16 +1735,12 @@ class Project < ActiveRecord::Base
     import_export_shared.archive_path
   end
-  def export_project_path
-    Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
-  end
-
   def export_status
     if export_in_progress?
       :started
     elsif after_export_in_progress?
       :after_export_action
-    elsif export_project_path || export_project_object_exists?
+    elsif export_file_exists?
       :finished
     else
       :none
@@ -1759,21 +1755,19 @@ class Project < ActiveRecord::Base
     import_export_shared.after_export_in_progress?
   end
-  def remove_exports(path = export_path)
-    if path.present?
-      FileUtils.rm_rf(path)
-    elsif export_project_object_exists?
-      import_export_upload.remove_export_file!
-      import_export_upload.save
-    end
+  def remove_exports
+    return unless export_file_exists?
+
+    import_export_upload.remove_export_file!
+    import_export_upload.save
   end
-  def remove_exported_project_file
-    remove_exports(export_project_path)
+  def export_file_exists?
+    export_file&.file
   end
-  def export_project_object_exists?
-    Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
+  def export_file
+    import_export_upload&.export_file
   end
   def full_path_slug

app/uploaders/avatar_uploader.rb

@@ -18,6 +18,10 @@ class AvatarUploader < GitlabUploader
     false
   end
+  def absolute_path
+    self.class.absolute_path(model.avatar)
+  end
+
   private
   def dynamic_segment

changelogs/unreleased/48778-remove-old-storage-logic-from-import-export.yml

+---
+title: Update Import/Export to only use new storage uploaders logic
+merge_request: 21409
+author:
+type: added

doc/administration/raketasks/project_import_export.md

@@ -9,6 +9,7 @@
 >    application settings (`/admin/application_settings`) under 'Import sources'.
 >  - The exports are stored in a temporary [shared directory][tmp] and are deleted
 >    every 24 hours by a specific worker.
+>  - ImportExport can use object storage automatically starting from GitLab 11.3 
 The GitLab Import/Export version can be checked by using:
@@ -30,12 +31,6 @@ sudo gitlab-rake gitlab:import_export:data
 bundle exec rake gitlab:import_export:data RAILS_ENV=production
 ```
-In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]:
-
-```
-import_export_object_storage
-``` 
-
 [ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
 [feature-flags]: https://docs.gitlab.com/ee/api/features.html
 [tmp]: ../../development/shared_files.md

lib/api/project_export.rb

@@ -21,12 +21,8 @@ module API
         detail 'This feature was introduced in GitLab 10.6.'
       end
       get ':id/export/download' do
-        path = user_project.export_project_path
-
-        if path
-          present_disk_file!(path, File.basename(path), 'application/gzip')
-        elsif user_project.export_project_object_exists?
-          present_carrierwave_file!(user_project.import_export_upload.export_file)
+        if user_project.export_file_exists?
+          present_carrierwave_file!(user_project.export_file)
         else
           render_api_error!('404 Not found or has expired', 404)
         end

lib/gitlab/import_export.rb

@@ -40,10 +40,6 @@ module Gitlab
       "#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
     end
-    def object_storage?
-      Feature.enabled?(:import_export_object_storage)
-    end
-
     def version
       VERSION
     end

lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb

@@ -53,7 +53,7 @@ module Gitlab
         end
         def self.lock_file_path(project)
-          return unless project.export_path || object_storage?
+          return unless project.export_path || export_file_exists?
           lock_path = project.import_export_shared.archive_path
@@ -83,8 +83,8 @@ module Gitlab
           errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
         end
-        def object_storage?
-          project.export_project_object_exists?
+        def export_file_exists?
+          project.export_file_exists?
         end
       end
     end

lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb

@@ -23,7 +23,7 @@ module Gitlab
         def strategy_execute
           handle_response_error(send_file)
-          project.remove_exported_project_file
+          project.remove_exports
         end
         def handle_response_error(response)
@@ -40,15 +40,11 @@ module Gitlab
         def send_file
           Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
         ensure
-          export_file.close if export_file && !object_storage?
+          export_file.close if export_file
         end
         def export_file
-          if object_storage?
-            project.import_export_upload.export_file.file.open
-          else
-            File.open(project.export_project_path)
-          end
+          project.export_file.open
         end
         def send_file_options
@@ -63,11 +59,7 @@ module Gitlab
         end
         def export_size
-          if object_storage?
-            project.import_export_upload.export_file.file.size
-          else
-            File.size(project.export_project_path)
-          end
+          project.export_file.file.size
         end
       end
     end

lib/gitlab/import_export/avatar_restorer.rb

@@ -19,7 +19,7 @@ module Gitlab
       private
       def avatar_export_file
-        @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first
+        @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first
       end
       def avatar_export_path

lib/gitlab/import_export/avatar_saver.rb

 module Gitlab
   module ImportExport
     class AvatarSaver
-      include Gitlab::ImportExport::CommandLineUtil
-
       def initialize(project:, shared:)
         @project = project
         @shared = shared
@@ -14,19 +12,12 @@ module Gitlab
         Gitlab::ImportExport::UploadsManager.new(
           project: @project,
           shared: @shared,
-          relative_export_path: 'avatar',
-          from: avatar_path
+          relative_export_path: 'avatar'
         ).save
       rescue => e
         @shared.error(e)
         false
       end
-
-      private
-
-      def avatar_path
-        @project.avatar.path
-      end
     end
   end
 end

lib/gitlab/import_export/importer.rb

@@ -92,8 +92,6 @@ module Gitlab
       end
       def remove_import_file
-        return unless Gitlab::ImportExport.object_storage?
-
         upload = @project.import_export_upload
         return unless upload&.import_file&.file

lib/gitlab/import_export/project_tree_restorer.rb

@@ -199,7 +199,7 @@ module Gitlab
       end
       def excluded_keys_for_relation(relation)
-        @reader.attributes_finder.find_excluded_keys(relation)
+        reader.attributes_finder.find_excluded_keys(relation)
       end
     end
   end

lib/gitlab/import_export/saver.rb

@@ -18,7 +18,7 @@ module Gitlab
           Rails.logger.info("Saved project export #{archive_file}")
-          save_on_object_storage if use_object_storage?
+          save_upload
         else
           @shared.error(Gitlab::ImportExport::Error.new(error_message))
           false
@@ -27,10 +27,8 @@ module Gitlab
         @shared.error(e)
         false
       ensure
-        if use_object_storage?
-          remove_archive
-          remove_export_path
-        end
+        remove_archive
+        remove_export_path
       end
       private
@@ -51,7 +49,7 @@ module Gitlab
         @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
       end
-      def save_on_object_storage
+      def save_upload
         upload = ImportExportUpload.find_or_initialize_by(project: @project)
         File.open(archive_file) { |file| upload.export_file = file }
@@ -59,12 +57,8 @@ module Gitlab
         upload.save!
       end
-      def use_object_storage?
-        Gitlab::ImportExport.object_storage?
-      end
-
       def error_message
-        "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
+        "Unable to save #{archive_file} into #{@shared.export_path}."
       end
     end
   end

lib/gitlab/import_export/uploads_manager.rb

@@ -5,18 +5,13 @@ module Gitlab
       UPLOADS_BATCH_SIZE = 100
-      def initialize(project:, shared:, relative_export_path: 'uploads', from: nil)
+      def initialize(project:, shared:, relative_export_path: 'uploads')
         @project = project
         @shared = shared
         @relative_export_path = relative_export_path
-        @from = from || default_uploads_path
       end
       def save
-        if File.file?(@from) && @relative_export_path == 'avatar'
-          copy_files(@from, File.join(uploads_export_path, @project.avatar.filename))
-        end
-
         copy_project_uploads
         true
@@ -55,17 +50,11 @@ module Gitlab
             copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path))
           else
-            next unless Gitlab::ImportExport.object_storage?
-
             download_and_copy(uploader)
           end
         end
       end
-      def default_uploads_path
-        FileUploader.absolute_base_dir(@project)
-      end
-
       def uploads_export_path
         @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path)
       end

lib/gitlab/import_export/uploads_restorer.rb

@@ -2,30 +2,14 @@ module Gitlab
   module ImportExport
     class UploadsRestorer < UploadsSaver
       def restore
-        if Gitlab::ImportExport.object_storage?
-          Gitlab::ImportExport::UploadsManager.new(
-            project: @project,
-            shared: @shared
-          ).restore
-        elsif File.directory?(uploads_export_path)
-          copy_files(uploads_export_path, uploads_path)
-
-          true
-        else
-          true # Proceed without uploads
-        end
+        Gitlab::ImportExport::UploadsManager.new(
+          project: @project,
+          shared: @shared
+        ).restore
       rescue => e
         @shared.error(e)
         false
       end
-
-      def uploads_path
-        FileUploader.absolute_base_dir(@project)
-      end
-
-      def uploads_export_path
-        @uploads_export_path ||= File.join(@shared.export_path, 'uploads')
-      end
     end
   end
 end

lib/gitlab/import_export/uploads_saver.rb

 module Gitlab
   module ImportExport
     class UploadsSaver
-      include Gitlab::ImportExport::CommandLineUtil
-
       def initialize(project:, shared:)
         @project = project
         @shared = shared

lib/gitlab/template_helper.rb

 module Gitlab
   module TemplateHelper
-    include Gitlab::Utils::StrongMemoize
-
     def prepare_template_environment(file)
       return unless file
-      if Gitlab::ImportExport.object_storage?
-        params[:import_export_upload] = ImportExportUpload.new(import_file: file)
-      else
-        FileUtils.mkdir_p(File.dirname(import_upload_path))
-        FileUtils.copy_entry(file.path, import_upload_path)
-
-        params[:import_source] = import_upload_path
-      end
-    end
-
-    def import_upload_path
-      strong_memoize(:import_upload_path) do
-        Gitlab::ImportExport.import_upload_path(filename: tmp_filename)
-      end
+      params[:import_export_upload] = ImportExportUpload.new(import_file: file)
     end
     def tmp_filename