Merge branch '40781-os-to-ce' into 'master'

Bring Object Storage to CE

Closes #4171, #4163, #3370, #2841, and #29203

See merge request gitlab-org/gitlab-ce!17358

app/controllers/concerns/send_file_upload.rb

+module SendFileUpload
+  def send_upload(file_upload, send_params: {}, redirect_params: {}, attachment: nil, disposition: 'attachment')
+    if attachment
+      redirect_params[:query] = { "response-content-disposition" => "#{disposition};filename=#{attachment.inspect}" }
+      send_params.merge!(filename: attachment, disposition: disposition)
+    end
+
+    if file_upload.file_storage?
+      send_file file_upload.path, send_params
+    elsif file_upload.class.proxy_download_enabled?
+      headers.store(*Gitlab::Workhorse.send_url(file_upload.url(**redirect_params)))
+      head :ok
+    else
+      redirect_to file_upload.url(**redirect_params)
+    end
+  end
+end

app/controllers/concerns/uploads_actions.rb

 module UploadsActions
   include Gitlab::Utils::StrongMemoize
+  include SendFileUpload
   UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo).freeze
@@ -26,14 +27,11 @@ module UploadsActions
   def show
     return render_404 unless uploader&.exists?
-    if uploader.file_storage?
-      disposition = uploader.image_or_video? ? 'inline' : 'attachment'
-      expires_in 0.seconds, must_revalidate: true, private: true
-      send_file uploader.file.path, disposition: disposition
-    else
-      redirect_to uploader.url
-    end
+    expires_in 0.seconds, must_revalidate: true, private: true
+    disposition = uploader.image_or_video? ? 'inline' : 'attachment'
+
+    send_upload(uploader, attachment: uploader.filename, disposition: disposition)
   end
   private
@@ -62,19 +60,27 @@ module UploadsActions
   end
   def build_uploader_from_upload
-    return nil unless params[:secret] && params[:filename]
-    upload_path = uploader_class.upload_path(params[:secret], params[:filename])
-    upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_path)
+    return unless uploader = build_uploader
+    upload_paths = uploader.upload_paths(params[:filename])
+    upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_paths)
     upload&.build_uploader
   end
   def build_uploader_from_params
+    return unless uploader = build_uploader
+
+    uploader.retrieve_from_store!(params[:filename])
+    uploader
+  end
+
+  def build_uploader
+    return unless params[:secret] && params[:filename]
+
     uploader = uploader_class.new(model, secret: params[:secret])
-    return nil unless uploader.model_valid?
-    uploader.retrieve_from_store!(params[:filename])
+    return unless uploader.model_valid?
     uploader
   end

app/controllers/projects/artifacts_controller.rb

 class Projects::ArtifactsController < Projects::ApplicationController
   include ExtractsPath
   include RendersBlob
+  include SendFileUpload
   layout 'project'
   before_action :authorize_read_build!
@@ -10,11 +11,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
   before_action :entry, only: [:file]
   def download
-    if artifacts_file.file_storage?
-      send_file artifacts_file.path, disposition: 'attachment'
-    else
-      redirect_to artifacts_file.url
-    end
+    send_upload(artifacts_file, attachment: artifacts_file.filename)
   end
   def browse
@@ -45,8 +42,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
   end
   def raw
-    path = Gitlab::Ci::Build::Artifacts::Path
-      .new(params[:path])
+    path = Gitlab::Ci::Build::Artifacts::Path.new(params[:path])
     send_artifacts_entry(build, path)
   end
@@ -75,7 +71,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
   end
   def validate_artifacts!
-    render_404 unless build && build.artifacts?
+    render_404 unless build&.artifacts?
   end
   def build

app/controllers/projects/jobs_controller.rb

 class Projects::JobsController < Projects::ApplicationController
+  include SendFileUpload
+
   before_action :build, except: [:index, :cancel_all]
   before_action :authorize_read_build!,
@@ -117,11 +119,17 @@ class Projects::JobsController < Projects::ApplicationController
   end
   def raw
-    build.trace.read do |stream|
-      if stream.file?
-        send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline'
-      else
-        render_404
+    if trace_artifact_file
+      send_upload(trace_artifact_file,
+                  send_params: raw_send_params,
+                  redirect_params: raw_redirect_params)
+    else
+      build.trace.read do |stream|
+        if stream.file?
+          send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline'
+        else
+          render_404
+        end
       end
     end
   end
@@ -136,9 +144,21 @@ class Projects::JobsController < Projects::ApplicationController
     return access_denied! unless can?(current_user, :erase_build, build)
   end
+  def raw_send_params
+    { type: 'text/plain; charset=utf-8', disposition: 'inline' }
+  end
+
+  def raw_redirect_params
+    { query: { 'response-content-type' => 'text/plain; charset=utf-8', 'response-content-disposition' => 'inline' } }
+  end
+
+  def trace_artifact_file
+    @trace_artifact_file ||= build.job_artifacts_trace&.file
+  end
+
   def build
     @build ||= project.builds.find(params[:id])
-      .present(current_user: current_user)
+                 .present(current_user: current_user)
   end
   def build_path(build)

app/controllers/projects/lfs_storage_controller.rb

 class Projects::LfsStorageController < Projects::GitHttpClientController
   include LfsRequest
   include WorkhorseRequest
+  include SendFileUpload
   skip_before_action :verify_workhorse_api!, only: [:download, :upload_finalize]
@@ -11,7 +12,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
       return
     end
-    send_file lfs_object.file.path, content_type: "application/octet-stream"
+    send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" })
   end
   def upload_authorize
@@ -70,10 +71,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
   end
   def move_tmp_file_to_storage(object, path)
-    File.open(path) do |f|
-      object.file = f
-    end
-
+    object.file = File.open(path)
     object.file.store!
     object.save
   end

app/controllers/projects/raw_controller.rb

@@ -2,6 +2,7 @@
 class Projects::RawController < Projects::ApplicationController
   include ExtractsPath
   include BlobHelper
+  include SendFileUpload
   before_action :require_non_empty_project
   before_action :assign_ref_vars
@@ -31,7 +32,7 @@ class Projects::RawController < Projects::ApplicationController
     lfs_object = find_lfs_object
     if lfs_object && lfs_object.project_allowed_access?(@project)
-      send_file lfs_object.file.path, filename: @blob.name, disposition: 'attachment'
+      send_upload(lfs_object.file, attachment: @blob.name)
     else
       render_404
     end

app/models/appearance.rb

 class Appearance < ActiveRecord::Base
   include CacheMarkdownField
+  include AfterCommitQueue
+  include ObjectStorage::BackgroundMove
   cache_markdown_field :description
   cache_markdown_field :new_project_guidelines

app/models/ci/build.rb

@@ -3,6 +3,7 @@ module Ci
     prepend ArtifactMigratable
     include TokenAuthenticatable
     include AfterCommitQueue
+    include ObjectStorage::BackgroundMove
     include Presentable
     include Importable
@@ -45,6 +46,7 @@ module Ci
       where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)',
         '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive)
     end
+    scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) }
     scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
     scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) }
     scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
@@ -365,13 +367,19 @@ module Ci
       project.running_or_pending_build_count(force: true)
     end
+    def browsable_artifacts?
+      artifacts_metadata?
+    end
+
     def artifacts_metadata_entry(path, **options)
-      metadata = Gitlab::Ci::Build::Artifacts::Metadata.new(
-        artifacts_metadata.path,
-        path,
-        **options)
-      metadata.to_entry
+      artifacts_metadata.use_file do |metadata_path|
+        metadata = Gitlab::Ci::Build::Artifacts::Metadata.new(
+          metadata_path,
+          path,
+          **options)
+        metadata.to_entry
+      end
     end
     def erase_artifacts!

app/models/ci/job_artifact.rb

 module Ci
   class JobArtifact < ActiveRecord::Base
+    include AfterCommitQueue
+    include ObjectStorage::BackgroundMove
     extend Gitlab::Ci::Model
     belongs_to :project
@@ -7,9 +9,11 @@ module Ci
     before_save :set_size, if: :file_changed?
+    scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
+
     mount_uploader :file, JobArtifactUploader
-    delegate :open, :exists?, to: :file
+    delegate :exists?, :open, to: :file
     enum file_type: {
       archive: 1,
@@ -21,6 +25,10 @@ module Ci
       self.where(project: project).sum(:size)
     end
+    def local_store?
+      [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store)
+    end
+
     def set_size
       self.size = file.size
     end

app/models/concerns/avatarable.rb

@@ -3,6 +3,7 @@ module Avatarable
   included do
     prepend ShadowMethods
+    include ObjectStorage::BackgroundMove
     validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
     validates :avatar, file_size: { maximum: 200.kilobytes.to_i }

app/models/lfs_object.rb

 class LfsObject < ActiveRecord::Base
+  include AfterCommitQueue
+  include ObjectStorage::BackgroundMove
+
   has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :projects, through: :lfs_objects_projects
+  scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) }
+
   validates :oid, presence: true, uniqueness: true
   mount_uploader :file, LfsObjectUploader
@@ -10,6 +15,10 @@ class LfsObject < ActiveRecord::Base
     projects.exists?(project.lfs_storage_project.id)
   end
+  def local_store?
+    [nil, LfsObjectUploader::Store::LOCAL].include?(self.file_store)
+  end
+
   def self.destroy_unreferenced
     joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id")
         .where(lfs_objects_projects: { id: nil })

app/models/upload.rb

@@ -9,6 +9,8 @@ class Upload < ActiveRecord::Base
   validates :model, presence: true
   validates :uploader, presence: true
+  scope :with_files_stored_locally, -> { where(store: [nil, ObjectStorage::Store::LOCAL]) }
+
   before_save  :calculate_checksum!, if: :foreground_checksummable?
   after_commit :schedule_checksum,   if: :checksummable?
@@ -21,6 +23,7 @@ class Upload < ActiveRecord::Base
   end
   def absolute_path
+    raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local?
     return path unless relative_path?
     uploader_class.absolute_path(self)
@@ -30,11 +33,11 @@ class Upload < ActiveRecord::Base
     self.checksum = nil
     return unless checksummable?
-    self.checksum = self.class.hexdigest(absolute_path)
+    self.checksum = Digest::SHA256.file(absolute_path).hexdigest
   end
-  def build_uploader
-    uploader_class.new(model, mount_point, **uploader_context).tap do |uploader|
+  def build_uploader(mounted_as = nil)
+    uploader_class.new(model, mounted_as || mount_point).tap do |uploader|
       uploader.upload = self
       uploader.retrieve_from_store!(identifier)
     end
@@ -51,6 +54,12 @@ class Upload < ActiveRecord::Base
     }.compact
   end
+  def local?
+    return true if store.nil?
+
+    store == ObjectStorage::Store::LOCAL
+  end
+
   private
   def delete_file!
@@ -61,10 +70,6 @@ class Upload < ActiveRecord::Base
     checksum.nil? && local? && exist?
   end
-  def local?
-    true
-  end
-
   def foreground_checksummable?
     checksummable? && size <= CHECKSUM_THRESHOLD
   end

app/services/projects/update_pages_service.rb

@@ -81,11 +81,13 @@ module Projects
     end
     def extract_tar_archive!(temp_path)
-      results = Open3.pipeline(%W(gunzip -c #{artifacts}),
-                               %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
-                               %W(tar -x -C #{temp_path} #{SITE_PATH}),
-                               err: '/dev/null')
-      raise FailedToExtractError, 'pages failed to extract' unless results.compact.all?(&:success?)
+      build.artifacts_file.use_file do |artifacts_path|
+        results = Open3.pipeline(%W(gunzip -c #{artifacts_path}),
+                                %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
+                                %W(tar -x -C #{temp_path} #{SITE_PATH}),
+                                err: '/dev/null')
+        raise FailedToExtractError, 'pages failed to extract' unless results.compact.all?(&:success?)
+      end
     end
     def extract_zip_archive!(temp_path)
@@ -103,8 +105,10 @@ module Projects
       # -n  never overwrite existing files
       # We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories
       site_path = File.join(SITE_PATH, '*')
-      unless system(*%W(unzip -qq -n #{artifacts} #{site_path} -d #{temp_path}))
-        raise FailedToExtractError, 'pages failed to extract'
+      build.artifacts_file.use_file do |artifacts_path|
+        unless system(*%W(unzip -n #{artifacts_path} #{site_path} -d #{temp_path}))
+          raise FailedToExtractError, 'pages failed to extract'
+        end
       end
     end

app/uploaders/attachment_uploader.rb

 class AttachmentUploader < GitlabUploader
-  include UploaderHelper
   include RecordsUploads::Concern
-
-  storage :file
+  include ObjectStorage::Concern
+  prepend ObjectStorage::Extension::RecordsUploads
+  include UploaderHelper
   private

app/uploaders/avatar_uploader.rb

 class AvatarUploader < GitlabUploader
   include UploaderHelper
   include RecordsUploads::Concern
-
-  storage :file
+  include ObjectStorage::Concern
+  prepend ObjectStorage::Extension::RecordsUploads
   def exists?
     model.avatar.file && model.avatar.file.present?
   end
-  def move_to_cache
+  def move_to_store
     false
   end
-  def move_to_store
+  def move_to_cache
     false
   end

app/uploaders/file_mover.rb

@@ -10,7 +10,11 @@ class FileMover
   def execute
     move
-    uploader.record_upload if update_markdown
+
+    if update_markdown
+      uploader.record_upload
+      uploader.schedule_background_upload
+    end
   end
   private
@@ -24,11 +28,8 @@ class FileMover
     updated_text = model.read_attribute(update_field)
                         .gsub(temp_file_uploader.markdown_link, uploader.markdown_link)
     model.update_attribute(update_field, updated_text)
-
-    true
   rescue
     revert
-
     false
   end

app/uploaders/file_uploader.rb

@@ -9,14 +9,18 @@
 class FileUploader < GitlabUploader
   include UploaderHelper
   include RecordsUploads::Concern
+  include ObjectStorage::Concern
+  prepend ObjectStorage::Extension::RecordsUploads
   MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)}
   DYNAMIC_PATH_PATTERN = %r{(?<secret>\h{32})/(?<identifier>.*)}
-  storage :file
-
   after :remove, :prune_store_dir
+  # FileUploader do not run in a model transaction, so we can simply
+  # enqueue a job after the :store hook.
+  after :store, :schedule_background_upload
+
   def self.root
     File.join(options.storage_path, 'uploads')
   end
@@ -28,8 +32,11 @@ class FileUploader < GitlabUploader
     )
   end
-  def self.base_dir(model)
-    model_path_segment(model)
+  def self.base_dir(model, store = Store::LOCAL)
+    decorated_model = model
+    decorated_model = Storage::HashedProject.new(model) if store == Store::REMOTE
+
+    model_path_segment(decorated_model)
   end
   # used in migrations and import/exports
@@ -47,21 +54,24 @@ class FileUploader < GitlabUploader
   #
   # Returns a String without a trailing slash
   def self.model_path_segment(model)
-    if model.hashed_storage?(:attachments)
-      model.disk_path
+    case model
+    when Storage::HashedProject then model.disk_path
     else
-      model.full_path
+      model.hashed_storage?(:attachments) ? model.disk_path : model.full_path
     end
   end
-  def self.upload_path(secret, identifier)
-    File.join(secret, identifier)
-  end
-
   def self.generate_secret
     SecureRandom.hex
   end
+  def upload_paths(filename)
+    [
+      File.join(secret, filename),
+      File.join(base_dir(Store::REMOTE), secret, filename)
+    ]
+  end
+
   attr_accessor :model
   def initialize(model, mounted_as = nil, **uploader_context)
@@ -71,8 +81,10 @@ class FileUploader < GitlabUploader
     apply_context!(uploader_context)
   end
-  def base_dir
-    self.class.base_dir(@model)
+  # enforce the usage of Hashed storage when storing to
+  # remote store as the FileMover doesn't support OS
+  def base_dir(store = nil)
+    self.class.base_dir(@model, store || object_store)
   end
   # we don't need to know the actual path, an uploader instance should be
@@ -82,15 +94,19 @@ class FileUploader < GitlabUploader
   end
   def upload_path
-    self.class.upload_path(dynamic_segment, identifier)
-  end
-
-  def model_path_segment
-    self.class.model_path_segment(@model)
+    if file_storage?
+      # Legacy path relative to project.full_path
+      File.join(dynamic_segment, identifier)
+    else
+      File.join(store_dir, identifier)
+    end
   end
-  def store_dir
-    File.join(base_dir, dynamic_segment)
+  def store_dirs
+    {
+      Store::LOCAL => File.join(base_dir, dynamic_segment),
+      Store::REMOTE => File.join(base_dir(ObjectStorage::Store::REMOTE), dynamic_segment)
+    }
   end
   def markdown_link

app/uploaders/gitlab_uploader.rb

@@ -37,12 +37,10 @@ class GitlabUploader < CarrierWave::Uploader::Base
     cache_storage.is_a?(CarrierWave::Storage::File)
   end
-  # Reduce disk IO
   def move_to_cache
     file_storage?
   end
-  # Reduce disk IO
   def move_to_store
     file_storage?
   end
@@ -51,10 +49,6 @@ class GitlabUploader < CarrierWave::Uploader::Base
     file.present?
   end
-  def store_dir
-    File.join(base_dir, dynamic_segment)
-  end
-
   def cache_dir
     File.join(root, base_dir, 'tmp/cache')
   end
@@ -76,6 +70,10 @@ class GitlabUploader < CarrierWave::Uploader::Base
   # Designed to be overridden by child uploaders that have a dynamic path
   # segment -- that is, a path that changes based on mutable attributes of its
   # associated model
+  #
+  # For example, `FileUploader` builds the storage path based on the associated
+  # project model's `path_with_namespace` value, which can change when the
+  # project or its containing namespace is moved or renamed.
   def dynamic_segment
     raise(NotImplementedError)
   end

app/uploaders/job_artifact_uploader.rb

 class JobArtifactUploader < GitlabUploader
   extend Workhorse::UploadPath
+  include ObjectStorage::Concern
   storage_options Gitlab.config.artifacts
@@ -14,9 +15,11 @@ class JobArtifactUploader < GitlabUploader
   end
   def open
-    raise 'Only File System is supported' unless file_storage?
-
-    File.open(path, "rb") if path
+    if file_storage?
+      File.open(path, "rb") if path
+    else
+      ::Gitlab::Ci::Trace::HttpIO.new(url, size) if url
+    end
   end
   private

app/uploaders/legacy_artifact_uploader.rb

 class LegacyArtifactUploader < GitlabUploader
   extend Workhorse::UploadPath
+  include ObjectStorage::Concern
   storage_options Gitlab.config.artifacts