Merge branch '40781-os-to-ce' into 'master'

Bring Object Storage to CE

Closes #4171, #4163, #3370, #2841, and #29203

See merge request gitlab-org/gitlab-ce!17358

spec/requests/api/jobs_spec.rb

 require 'spec_helper'
 describe API::Jobs do
+  include HttpIOHelpers
+
   set(:project) do
     create(:project, :repository, public_builds: false)
   end
@@ -112,6 +114,7 @@ describe API::Jobs do
     let(:query) { Hash.new }
     before do
+      job
       get api("/projects/#{project.id}/pipelines/#{pipeline.id}/jobs", api_user), query
     end
@@ -335,10 +338,55 @@ describe API::Jobs do
           end
         end
+        context 'when artifacts are stored remotely' do
+          let(:proxy_download) { false }
+
+          before do
+            stub_artifacts_object_storage(proxy_download: proxy_download)
+          end
+
+          let(:job) { create(:ci_build, pipeline: pipeline) }
+          let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) }
+
+          before do
+            job.reload
+
+            get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
+          end
+
+          context 'when proxy download is enabled' do
+            let(:proxy_download) { true }
+
+            it 'responds with the workhorse send-url' do
+              expect(response.headers[Gitlab::Workhorse::SEND_DATA_HEADER]).to start_with("send-url:")
+            end
+          end
+
+          context 'when proxy download is disabled' do
+            it 'returns location redirect' do
+              expect(response).to have_gitlab_http_status(302)
+            end
+          end
+
+          context 'authorized user' do
+            it 'returns the file remote URL' do
+              expect(response).to redirect_to(artifact.file.url)
+            end
+          end
+
+          context 'unauthorized user' do
+            let(:api_user) { nil }
+
+            it 'does not return specific job artifacts' do
+              expect(response).to have_gitlab_http_status(404)
+            end
+          end
+        end
+
         it 'does not return job artifacts if not uploaded' do
           get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
-          expect(response).to have_gitlab_http_status(404)
+          expect(response).to have_gitlab_http_status(:not_found)
         end
       end
     end
@@ -349,6 +397,7 @@ describe API::Jobs do
     let(:job) { create(:ci_build, :artifacts, pipeline: pipeline, user: api_user) }
     before do
+      stub_artifacts_object_storage
       job.success
     end
@@ -412,9 +461,24 @@ describe API::Jobs do
                 "attachment; filename=#{job.artifacts_file.filename}" }
           end
-          it { expect(response).to have_gitlab_http_status(200) }
+          it { expect(response).to have_http_status(:ok) }
           it { expect(response.headers).to include(download_headers) }
         end
+
+        context 'when artifacts are stored remotely' do
+          let(:job) { create(:ci_build, pipeline: pipeline, user: api_user) }
+          let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) }
+
+          before do
+            job.reload
+
+            get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
+          end
+
+          it 'returns location redirect' do
+            expect(response).to have_http_status(:found)
+          end
+        end
       end
       context 'with regular branch' do
@@ -451,6 +515,22 @@ describe API::Jobs do
     end
     context 'authorized user' do
+      context 'when trace is in ObjectStorage' do
+        let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+        before do
+          stub_remote_trace_206
+          allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+          allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url }
+          allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size }
+        end
+
+        it 'returns specific job trace' do
+          expect(response).to have_gitlab_http_status(200)
+          expect(response.body).to eq(job.trace.raw)
+        end
+      end
+
       context 'when trace is artifact' do
         let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }

spec/requests/api/runner_spec.rb

@@ -200,7 +200,7 @@ describe API::Runner do
     let(:project) { create(:project, shared_runners_enabled: false) }
     let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') }
     let(:runner) { create(:ci_runner) }
-    let!(:job) do
+    let(:job) do
       create(:ci_build, :artifacts, :extended_options,
              pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0, commands: "ls\ndate")
     end
@@ -215,6 +215,7 @@ describe API::Runner do
       let(:user_agent) { 'gitlab-runner 9.0.0 (9-0-stable; go1.7.4; linux/amd64)' }
       before do
+        job
         stub_container_registry_config(enabled: false)
       end
@@ -888,6 +889,7 @@ describe API::Runner do
       let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') }
       before do
+        stub_artifacts_object_storage
         job.run!
       end
@@ -1179,27 +1181,67 @@ describe API::Runner do
       describe 'GET /api/v4/jobs/:id/artifacts' do
         let(:token) { job.token }
-        before do
-          download_artifact
-        end
-
         context 'when job has artifacts' do
-          let(:job) { create(:ci_build, :artifacts) }
-          let(:download_headers) do
-            { 'Content-Transfer-Encoding' => 'binary',
-              'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' }
+          let(:job) { create(:ci_build) }
+          let(:store) { JobArtifactUploader::Store::LOCAL }
+
+          before do
+            create(:ci_job_artifact, :archive, file_store: store, job: job)
           end
           context 'when using job token' do
-            it 'download artifacts' do
-              expect(response).to have_gitlab_http_status(200)
-              expect(response.headers).to include download_headers
+            context 'when artifacts are stored locally' do
+              let(:download_headers) do
+                { 'Content-Transfer-Encoding' => 'binary',
+                  'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' }
+              end
+
+              before do
+                download_artifact
+              end
+
+              it 'download artifacts' do
+                expect(response).to have_http_status(200)
+                expect(response.headers).to include download_headers
+              end
+            end
+
+            context 'when artifacts are stored remotely' do
+              let(:store) { JobArtifactUploader::Store::REMOTE }
+              let!(:job) { create(:ci_build) }
+
+              context 'when proxy download is being used' do
+                before do
+                  download_artifact(direct_download: false)
+                end
+
+                it 'uses workhorse send-url' do
+                  expect(response).to have_gitlab_http_status(200)
+                  expect(response.headers).to include(
+                    'Gitlab-Workhorse-Send-Data' => /send-url:/)
+                end
+              end
+
+              context 'when direct download is being used' do
+                before do
+                  download_artifact(direct_download: true)
+                end
+
+                it 'receive redirect for downloading artifacts' do
+                  expect(response).to have_gitlab_http_status(302)
+                  expect(response.headers).to include('Location')
+                end
+              end
             end
           end
           context 'when using runnners token' do
             let(:token) { job.project.runners_token }
+            before do
+              download_artifact
+            end
+
             it 'responds with forbidden' do
               expect(response).to have_gitlab_http_status(403)
             end
@@ -1208,12 +1250,16 @@ describe API::Runner do
         context 'when job does not has artifacts' do
           it 'responds with not found' do
+            download_artifact
+
             expect(response).to have_gitlab_http_status(404)
           end
         end
         def download_artifact(params = {}, request_headers = headers)
           params = params.merge(token: token)
+          job.reload
+
           get api("/jobs/#{job.id}/artifacts"), params, request_headers
         end
       end

spec/requests/api/v3/builds_spec.rb

@@ -216,6 +216,7 @@ describe API::V3::Builds do
   describe 'GET /projects/:id/builds/:build_id/artifacts' do
     before do
+      stub_artifacts_object_storage
       get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user)
     end
@@ -230,13 +231,24 @@ describe API::V3::Builds do
           end
           it 'returns specific job artifacts' do
-            expect(response).to have_gitlab_http_status(200)
+            expect(response).to have_http_status(200)
             expect(response.headers).to include(download_headers)
             expect(response.body).to match_file(build.artifacts_file.file.file)
           end
         end
       end
+      context 'when artifacts are stored remotely' do
+        let(:build) { create(:ci_build, pipeline: pipeline) }
+        let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: build) }
+
+        it 'returns location redirect' do
+          get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user)
+
+          expect(response).to have_gitlab_http_status(302)
+        end
+      end
+
       context 'unauthorized user' do
         let(:api_user) { nil }
@@ -256,6 +268,7 @@ describe API::V3::Builds do
     let(:build) { create(:ci_build, :artifacts, pipeline: pipeline) }
     before do
+      stub_artifacts_object_storage
       build.success
     end
@@ -318,9 +331,24 @@ describe API::V3::Builds do
                 "attachment; filename=#{build.artifacts_file.filename}" }
           end
-          it { expect(response).to have_gitlab_http_status(200) }
+          it { expect(response).to have_http_status(200) }
           it { expect(response.headers).to include(download_headers) }
         end
+
+        context 'when artifacts are stored remotely' do
+          let(:build) { create(:ci_build, pipeline: pipeline) }
+          let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: build) }
+
+          before do
+            build.reload
+
+            get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user)
+          end
+
+          it 'returns location redirect' do
+            expect(response).to have_http_status(302)
+          end
+        end
       end
       context 'with regular branch' do

spec/requests/lfs_http_spec.rb

@@ -191,10 +191,12 @@ describe 'Git LFS API and storage' do
   describe 'when fetching lfs object' do
     let(:project) { create(:project) }
     let(:update_permissions) { }
+    let(:before_get) { }
     before do
       enable_lfs
       update_permissions
+      before_get
       get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers
     end
@@ -239,6 +241,21 @@ describe 'Git LFS API and storage' do
             end
             it_behaves_like 'responds with a file'
+
+            context 'when LFS uses object storage' do
+              let(:before_get) do
+                stub_lfs_object_storage
+                lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+              end
+
+              it 'responds with redirect' do
+                expect(response).to have_gitlab_http_status(302)
+              end
+
+              it 'responds with the file location' do
+                expect(response.location).to include(lfs_object.reload.file.path)
+              end
+            end
           end
         end
@@ -978,6 +995,32 @@ describe 'Git LFS API and storage' do
             end
           end
+          context 'and workhorse requests upload finalize for a new lfs object' do
+            before do
+              lfs_object.destroy
+            end
+
+            context 'with object storage disabled' do
+              it "doesn't attempt to migrate file to object storage" do
+                expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+                put_finalize(with_tempfile: true)
+              end
+            end
+
+            context 'with object storage enabled' do
+              before do
+                stub_lfs_object_storage(background_upload: true)
+              end
+
+              it 'schedules migration of file to object storage' do
+                expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('LfsObjectUploader', 'LfsObject', :file, kind_of(Numeric))
+
+                put_finalize(with_tempfile: true)
+              end
+            end
+          end
+
           context 'invalid tempfiles' do
             it 'rejects slashes in the tempfile name (path traversal' do
               put_finalize('foo/bar')
@@ -1177,7 +1220,9 @@ describe 'Git LFS API and storage' do
       put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}/authorize", nil, authorize_headers
     end
-    def put_finalize(lfs_tmp = lfs_tmp_file)
+    def put_finalize(lfs_tmp = lfs_tmp_file, with_tempfile: false)
+      setup_tempfile(lfs_tmp) if with_tempfile
+
       put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}", nil,
           headers.merge('X-Gitlab-Lfs-Tmp' => lfs_tmp).compact
     end
@@ -1185,6 +1230,13 @@ describe 'Git LFS API and storage' do
     def lfs_tmp_file
       "#{sample_oid}012345678"
     end
+
+    def setup_tempfile(lfs_tmp)
+      upload_path = LfsObjectUploader.workhorse_upload_path
+
+      FileUtils.mkdir_p(upload_path)
+      FileUtils.touch(File.join(upload_path, lfs_tmp))
+    end
   end
   def enable_lfs

spec/serializers/pipeline_serializer_spec.rb

@@ -117,6 +117,7 @@ describe PipelineSerializer do
       shared_examples 'no N+1 queries' do
         it 'verifies number of queries', :request_store do
           recorded = ActiveRecord::QueryRecorder.new { subject }
+
           expect(recorded.count).to be_within(1).of(36)
           expect(recorded.cached_count).to eq(0)
         end

spec/services/ci/retry_build_service_spec.rb

@@ -28,7 +28,8 @@ describe Ci::RetryBuildService do
     %i[type lock_version target_url base_tags trace_sections
        commit_id deployments erased_by_id last_deployment project_id
        runner_id tag_taggings taggings tags trigger_request_id
-       user_id auto_canceled_by_id retried failure_reason].freeze
+       user_id auto_canceled_by_id retried failure_reason
+       artifacts_file_store artifacts_metadata_store].freeze
   shared_examples 'build duplication' do
     let(:another_pipeline) { create(:ci_empty_pipeline, project: project) }

spec/services/issues/move_service_spec.rb

@@ -6,7 +6,7 @@ describe Issues::MoveService do
   let(:title) { 'Some issue' }
   let(:description) { 'Some issue description' }
   let(:old_project) { create(:project) }
-  let(:new_project) { create(:project, group: create(:group)) }
+  let(:new_project) { create(:project) }
   let(:milestone1) { create(:milestone, project_id: old_project.id, title: 'v9.0') }
   let(:old_issue) do

spec/support/http_io/http_io_helpers.rb

+module HttpIOHelpers
+  def stub_remote_trace_206
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return { |request| remote_trace_response(request, 206) }
+  end
+
+  def stub_remote_trace_200
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return { |request| remote_trace_response(request, 200) }
+  end
+
+  def stub_remote_trace_500
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return(status: [500, "Internal Server Error"])
+  end
+
+  def remote_trace_url
+    "http://trace.com/trace"
+  end
+
+  def remote_trace_response(request, responce_status)
+    range = request.headers['Range'].match(/bytes=(\d+)-(\d+)/)
+
+    {
+      status: responce_status,
+      headers: remote_trace_response_headers(responce_status, range[1].to_i, range[2].to_i),
+      body: range_trace_body(range[1].to_i, range[2].to_i)
+    }
+  end
+
+  def remote_trace_response_headers(responce_status, from, to)
+    headers = { 'Content-Type' => 'text/plain' }
+
+    if responce_status == 206
+      headers.merge('Content-Range' => "bytes #{from}-#{to}/#{remote_trace_size}")
+    end
+
+    headers
+  end
+
+  def range_trace_body(from, to)
+    remote_trace_body[from..to]
+  end
+
+  def remote_trace_body
+    @remote_trace_body ||= File.read(expand_fixture_path('trace/sample_trace'))
+  end
+
+  def remote_trace_size
+    remote_trace_body.length
+  end
+
+  def set_smaller_buffer_size_than(file_size)
+    blocks = (file_size / 128)
+    new_size = (blocks / 2) * 128
+    stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size)
+  end
+
+  def set_larger_buffer_size_than(file_size)
+    blocks = (file_size / 128)
+    new_size = (blocks * 2) * 128
+    stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size)
+  end
+end

spec/support/shared_examples/uploaders/object_storage_shared_examples.rb

+shared_context 'with storage' do |store, **stub_params|
+  before do
+    subject.object_store = store
+  end
+end
+
+shared_examples "migrates" do |to_store:, from_store: nil|
+  let(:to) { to_store }
+  let(:from) { from_store || subject.object_store }
+
+  def migrate(to)
+    subject.migrate!(to)
+  end
+
+  def checksum
+    Digest::SHA256.hexdigest(subject.read)
+  end
+
+  before do
+    migrate(from)
+  end
+
+  it 'returns corresponding file type' do
+    expect(subject).to be_an(CarrierWave::Uploader::Base)
+    expect(subject).to be_a(ObjectStorage::Concern)
+
+    if from == described_class::Store::REMOTE
+      expect(subject.file).to be_a(CarrierWave::Storage::Fog::File)
+    elsif from == described_class::Store::LOCAL
+      expect(subject.file).to be_a(CarrierWave::SanitizedFile)
+    else
+      raise 'Unexpected file type'
+    end
+  end
+
+  it 'does nothing when migrating to the current store' do
+    expect { migrate(from) }.not_to change { subject.object_store }.from(from)
+  end
+
+  it 'migrate to the specified store' do
+    from_checksum = checksum
+
+    expect { migrate(to) }.to change { subject.object_store }.from(from).to(to)
+    expect(checksum).to eq(from_checksum)
+  end
+
+  it 'removes the original file after the migration' do
+    original_file = subject.file.path
+    migrate(to)
+
+    expect(File.exist?(original_file)).to be_falsey
+  end
+
+  it 'can access to the original file during migration' do
+    file = subject.file
+
+    allow(subject).to receive(:delete_migrated_file) { } # Remove as a callback of :migrate
+    allow(subject).to receive(:record_upload) { } # Remove as a callback of :store (:record_upload)
+
+    expect(file.exists?).to be_truthy
+    expect { migrate(to) }.not_to change { file.exists? }
+  end
+
+  context 'when migrate! is not oqqupied by another process' do
+    it 'executes migrate!' do
+      expect(subject).to receive(:object_store=).at_least(1)
+
+      migrate(to)
+    end
+  end
+
+  context 'when migrate! is occupied by another process' do
+    let(:exclusive_lease_key) { "object_storage_migrate:#{subject.model.class}:#{subject.model.id}" }
+
+    before do
+      @uuid = Gitlab::ExclusiveLease.new(exclusive_lease_key, timeout: 1.hour.to_i).try_obtain
+    end
+
+    it 'does not execute migrate!' do
+      expect(subject).not_to receive(:unsafe_migrate!)
+
+      expect { migrate(to) }.to raise_error('Already running')
+    end
+
+    after do
+      Gitlab::ExclusiveLease.cancel(exclusive_lease_key, @uuid)
+    end
+  end
+
+  context 'migration is unsuccessful' do
+    shared_examples "handles gracefully" do |error:|
+      it 'does not update the object_store' do
+        expect { migrate(to) }.to raise_error(error)
+        expect(subject.object_store).to eq(from)
+      end
+
+      it 'does not delete the original file' do
+        expect { migrate(to) }.to raise_error(error)
+        expect(subject.exists?).to be_truthy
+      end
+    end
+
+    context 'when the store is not supported' do
+      let(:to) { -1 } # not a valid store
+
+      include_examples "handles gracefully", error: ObjectStorage::UnknownStoreError
+    end
+
+    context 'upon a fog failure' do
+      before do
+        storage_class = subject.send(:storage_for, to).class
+        expect_any_instance_of(storage_class).to receive(:store!).and_raise("Store failure.")
+      end
+
+      include_examples "handles gracefully", error: "Store failure."
+    end
+
+    context 'upon a database failure' do
+      before do
+        expect(uploader).to receive(:persist_object_store!).and_raise("ActiveRecord failure.")
+      end
+
+      include_examples "handles gracefully", error: "ActiveRecord failure."
+    end
+  end
+end

spec/support/stub_object_storage.rb

+module StubConfiguration
+  def stub_object_storage_uploader(
+        config:, uploader:, remote_directory:,
+        enabled: true,
+        proxy_download: false,
+        background_upload: false)
+    Fog.mock!
+
+    allow(config).to receive(:enabled) { enabled }
+    allow(config).to receive(:proxy_download) { proxy_download }
+    allow(config).to receive(:background_upload) { background_upload }
+
+    return unless enabled
+
+    ::Fog::Storage.new(uploader.object_store_credentials).tap do |connection|
+      begin
+        connection.directories.create(key: remote_directory)
+      rescue Excon::Error::Conflict
+      end
+    end
+  end
+
+  def stub_artifacts_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.artifacts.object_store,
+                                 uploader: JobArtifactUploader,
+                                 remote_directory: 'artifacts',
+                                 **params)
+  end
+
+  def stub_lfs_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.lfs.object_store,
+                                 uploader: LfsObjectUploader,
+                                 remote_directory: 'lfs-objects',
+                                 **params)
+  end
+
+  def stub_uploads_object_storage(uploader = described_class, **params)
+    stub_object_storage_uploader(config: Gitlab.config.uploads.object_store,
+                                 uploader: uploader,
+                                 remote_directory: 'uploads',
+                                 **params)
+  end
+end

spec/tasks/gitlab/artifacts/migrate_rake_spec.rb

+require 'rake_helper'
+
+describe 'gitlab:artifacts namespace rake task' do
+  before(:context) do
+    Rake.application.rake_require 'tasks/gitlab/artifacts/migrate'
+  end
+
+  let(:object_storage_enabled) { false }
+
+  before do
+    stub_artifacts_object_storage(enabled: object_storage_enabled)
+  end
+
+  subject { run_rake_task('gitlab:artifacts:migrate') }
+
+  context 'legacy artifacts' do
+    describe 'migrate' do
+      let!(:build) { create(:ci_build, :legacy_artifacts, artifacts_file_store: store, artifacts_metadata_store: store) }
+
+      context 'when local storage is used' do
+        let(:store) { ObjectStorage::Store::LOCAL }
+
+        context 'and job does not have file store defined' do
+          let(:object_storage_enabled) { true }
+          let(:store) { nil }
+
+          it "migrates file to remote storage" do
+            subject
+
+            expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+            expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+          end
+        end
+
+        context 'and remote storage is defined' do
+          let(:object_storage_enabled) { true }
+
+          it "migrates file to remote storage" do
+            subject
+
+            expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+            expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+          end
+        end
+
+        context 'and remote storage is not defined' do
+          it "fails to migrate to remote storage" do
+            subject
+
+            expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::LOCAL)
+            expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::LOCAL)
+          end
+        end
+      end
+
+      context 'when remote storage is used' do
+        let(:object_storage_enabled) { true }
+
+        let(:store) { ObjectStorage::Store::REMOTE }
+
+        it "file stays on remote storage" do
+          subject
+
+          expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+          expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+    end
+  end
+
+  context 'job artifacts' do
+    let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) }
+
+    context 'when local storage is used' do
+      let(:store) { ObjectStorage::Store::LOCAL }
+
+      context 'and job does not have file store defined' do
+        let(:object_storage_enabled) { true }
+        let(:store) { nil }
+
+        it "migrates file to remote storage" do
+          subject
+
+          expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+
+      context 'and remote storage is defined' do
+        let(:object_storage_enabled) { true }
+
+        it "migrates file to remote storage" do
+          subject
+
+          expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+
+      context 'and remote storage is not defined' do
+        it "fails to migrate to remote storage" do
+          subject
+
+          expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        end
+      end
+    end
+
+    context 'when remote storage is used' do
+      let(:object_storage_enabled) { true }
+      let(:store) { ObjectStorage::Store::REMOTE }
+
+      it "file stays on remote storage" do
+        subject
+
+        expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+      end
+    end
+  end
+end

spec/tasks/gitlab/lfs/migrate_rake_spec.rb

+require 'rake_helper'
+
+describe 'gitlab:lfs namespace rake task' do
+  before :all do
+    Rake.application.rake_require 'tasks/gitlab/lfs/migrate'
+  end
+
+  describe 'migrate' do
+    let(:local) { ObjectStorage::Store::LOCAL }
+    let(:remote) { ObjectStorage::Store::REMOTE }
+    let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) }
+
+    def lfs_migrate
+      run_rake_task('gitlab:lfs:migrate')
+    end
+
+    context 'object storage disabled' do
+      before do
+        stub_lfs_object_storage(enabled: false)
+      end
+
+      it "doesn't migrate files" do
+        expect { lfs_migrate }.not_to change { lfs_object.reload.file_store }
+      end
+    end
+
+    context 'object storage enabled' do
+      before do
+        stub_lfs_object_storage
+      end
+
+      it 'migrates local file to object storage' do
+        expect { lfs_migrate }.to change { lfs_object.reload.file_store }.from(local).to(remote)
+      end
+    end
+  end
+end

spec/tasks/gitlab/uploads/migrate_rake_spec.rb

+require 'rake_helper'
+
+describe 'gitlab:uploads:migrate rake tasks' do
+  let!(:projects) { create_list(:project, 10, :with_avatar) }
+  let(:model_class) { Project }
+  let(:uploader_class) { AvatarUploader }
+  let(:mounted_as) { :avatar }
+  let(:batch_size) { 3 }
+
+  before do
+    stub_env('BATCH', batch_size.to_s)
+    stub_uploads_object_storage(uploader_class)
+    Rake.application.rake_require 'tasks/gitlab/uploads/migrate'
+
+    allow(ObjectStorage::MigrateUploadsWorker).to receive(:perform_async)
+  end
+
+  def run
+    args = [uploader_class.to_s, model_class.to_s, mounted_as].compact
+    run_rake_task("gitlab:uploads:migrate", *args)
+  end
+
+  it 'enqueue jobs in batch' do
+    expect(ObjectStorage::MigrateUploadsWorker).to receive(:enqueue!).exactly(4).times
+
+    run
+  end
+end

spec/uploaders/attachment_uploader_spec.rb

@@ -11,4 +11,26 @@ describe AttachmentUploader do
                   store_dir: %r[uploads/-/system/note/attachment/],
                   upload_path: %r[uploads/-/system/note/attachment/],
                   absolute_path: %r[#{CarrierWave.root}/uploads/-/system/note/attachment/]
+
+  context "object_store is REMOTE" do
+    before do
+      stub_uploads_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like 'builds correct paths',
+                    store_dir: %r[note/attachment/],
+                    upload_path: %r[note/attachment/]
+  end
+
+  describe "#migrate!" do
+    before do
+      uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt')))
+      stub_uploads_object_storage
+    end
+
+    it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+    it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+  end
 end

spec/uploaders/avatar_uploader_spec.rb

 require 'spec_helper'
 describe AvatarUploader do
-  let(:model) { create(:user, :with_avatar) }
+  let(:model) { build_stubbed(:user) }
   let(:uploader) { described_class.new(model, :avatar) }
   let(:upload) { create(:upload, model: model) }
@@ -12,15 +12,28 @@ describe AvatarUploader do
                   upload_path: %r[uploads/-/system/user/avatar/],
                   absolute_path: %r[#{CarrierWave.root}/uploads/-/system/user/avatar/]
-  describe '#move_to_cache' do
-    it 'is false' do
-      expect(uploader.move_to_cache).to eq(false)
+  context "object_store is REMOTE" do
+    before do
+      stub_uploads_object_storage
     end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like 'builds correct paths',
+                    store_dir: %r[user/avatar/],
+                    upload_path: %r[user/avatar/]
   end
-  describe '#move_to_store' do
-    it 'is false' do
-      expect(uploader.move_to_store).to eq(false)
+  context "with a file" do
+    let(:project) { create(:project, :with_avatar) }
+    let(:uploader) { project.avatar }
+    let(:upload) { uploader.upload }
+
+    before do
+      stub_uploads_object_storage
     end
+
+    it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+    it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
   end
 end

spec/uploaders/file_mover_spec.rb

@@ -36,6 +36,12 @@ describe FileMover do
       it 'creates a new update record' do
         expect { subject }.to change { Upload.count }.by(1)
       end
+
+      it 'schedules a background migration' do
+        expect_any_instance_of(PersonalFileUploader).to receive(:schedule_background_upload).once
+
+        subject
+      end
     end
     context 'when update_markdown fails' do

spec/uploaders/file_uploader_spec.rb

@@ -11,32 +11,41 @@ describe FileUploader do
   shared_examples 'builds correct legacy storage paths' do
     include_examples 'builds correct paths',
                      store_dir: %r{awesome/project/\h+},
+                     upload_path: %r{\h+/<filename>},
                      absolute_path: %r{#{described_class.root}/awesome/project/secret/foo.jpg}
   end
-  shared_examples 'uses hashed storage' do
-    context 'when rolled out attachments' do
-      let(:project) { build_stubbed(:project, namespace: group, name: 'project') }
-      before do
-        allow(project).to receive(:disk_path).and_return('ca/fe/fe/ed')
-      end
-      it_behaves_like 'builds correct paths',
-                      store_dir: %r{ca/fe/fe/ed/\h+},
-                      absolute_path: %r{#{described_class.root}/ca/fe/fe/ed/secret/foo.jpg}
-    end
-    context 'when only repositories are rolled out' do
-      let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) }
-      it_behaves_like 'builds correct legacy storage paths'
+  context 'legacy storage' do
+    it_behaves_like 'builds correct legacy storage paths'
+    context 'uses hashed storage' do
+      context 'when rolled out attachments' do
+        let(:project) { build_stubbed(:project, namespace: group, name: 'project') }
+        include_examples 'builds correct paths',
+                         store_dir: %r{@hashed/\h{2}/\h{2}/\h+},
+                         upload_path: %r{\h+/<filename>}
+      end
+      context 'when only repositories are rolled out' do
+        let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) }
+        it_behaves_like 'builds correct legacy storage paths'
+      end
     end
   end
-  context 'legacy storage' do
-    it_behaves_like 'builds correct legacy storage paths'
-    include_examples 'uses hashed storage'
+  context 'object store is remote' do
+    before do
+      stub_uploads_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    # always use hashed storage path for remote uploads
+    it_behaves_like 'builds correct paths',
+                     store_dir: %r{@hashed/\h{2}/\h{2}/\h+},
+                     upload_path: %r{@hashed/\h{2}/\h{2}/\h+/\h+/<filename>}
   end
   describe 'initialize' do
@@ -78,6 +87,16 @@ describe FileUploader do
     end
   end
+  describe "#migrate!" do
+    before do
+      uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/dk.png')))
+      stub_uploads_object_storage
+    end
+
+    it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+    it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+  end
+
   describe '#upload=' do
     let(:secret) { SecureRandom.hex }
     let(:upload) { create(:upload, :issuable_upload, secret: secret, filename: 'file.txt') }
@@ -93,15 +112,5 @@ describe FileUploader do
       uploader.upload = upload
     end
-
-    context 'uploader_context is empty' do
-      it 'fallbacks to regex based extraction' do
-        expect(upload).to receive(:uploader_context).and_return({})
-
-        uploader.upload = upload
-        expect(uploader.secret).to eq(secret)
-        expect(uploader.instance_variable_get(:@identifier)).to eq('file.txt')
-      end
-    end
   end
 end

spec/uploaders/job_artifact_uploader_spec.rb

 require 'spec_helper'
 describe JobArtifactUploader do
-  let(:job_artifact) { create(:ci_job_artifact) }
+  let(:store) { described_class::Store::LOCAL }
+  let(:job_artifact) { create(:ci_job_artifact, file_store: store) }
   let(:uploader) { described_class.new(job_artifact, :file) }
   subject { uploader }
@@ -11,6 +12,17 @@ describe JobArtifactUploader do
                   cache_dir: %r[artifacts/tmp/cache],
                   work_dir: %r[artifacts/tmp/work]
+  context "object store is REMOTE" do
+    before do
+      stub_artifacts_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like "builds correct paths",
+                    store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z]
+  end
+
   describe '#open' do
     subject { uploader.open }
@@ -36,6 +48,17 @@ describe JobArtifactUploader do
         end
       end
     end
+
+    context 'when trace is stored in Object storage' do
+      before do
+        allow(uploader).to receive(:file_storage?) { false }
+        allow(uploader).to receive(:url) { 'http://object_storage.com/trace' }
+      end
+
+      it 'returns http io stream' do
+        is_expected.to be_a(Gitlab::Ci::Trace::HttpIO)
+      end
+    end
   end
   context 'file is stored in valid local_path' do
@@ -55,4 +78,14 @@ describe JobArtifactUploader do
     it { is_expected.to include("/#{job_artifact.job_id}/#{job_artifact.id}/") }
     it { is_expected.to end_with("ci_build_artifacts.zip") }
   end
+
+  describe "#migrate!" do
+    before do
+      uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/trace/sample_trace')))
+      stub_artifacts_object_storage
+    end
+
+    it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+    it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+  end
 end

spec/uploaders/legacy_artifact_uploader_spec.rb

 require 'rails_helper'
 describe LegacyArtifactUploader do
-  let(:job) { create(:ci_build) }
+  let(:store) { described_class::Store::LOCAL }
+  let(:job) { create(:ci_build, artifacts_file_store: store) }
   let(:uploader) { described_class.new(job, :legacy_artifacts_file) }
   let(:local_path) { described_class.root }
@@ -20,6 +21,17 @@ describe LegacyArtifactUploader do
                   cache_dir: %r[artifacts/tmp/cache],
                   work_dir: %r[artifacts/tmp/work]
+  context 'object store is remote' do
+    before do
+      stub_artifacts_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like "builds correct paths",
+                    store_dir: %r[\d{4}_\d{1,2}/\d+/\d+\z]
+  end
+
   describe '#filename' do
     # we need to use uploader, as this makes to use mounter
     # which initialises uploader.file object

spec/uploaders/lfs_object_uploader_spec.rb

@@ -11,4 +11,62 @@ describe LfsObjectUploader do
                   store_dir: %r[\h{2}/\h{2}],
                   cache_dir: %r[/lfs-objects/tmp/cache],
                   work_dir: %r[/lfs-objects/tmp/work]
+
+  context "object store is REMOTE" do
+    before do
+      stub_lfs_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like "builds correct paths",
+                    store_dir: %r[\h{2}/\h{2}]
+  end
+
+  describe 'migration to object storage' do
+    context 'with object storage disabled' do
+      it "is skipped" do
+        expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+        lfs_object
+      end
+    end
+
+    context 'with object storage enabled' do
+      before do
+        stub_lfs_object_storage(background_upload: true)
+      end
+
+      it 'is scheduled to run after creation' do
+        expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with(described_class.name, 'LfsObject', :file, kind_of(Numeric))
+
+        lfs_object
+      end
+    end
+  end
+
+  describe 'remote file' do
+    let(:remote) { described_class::Store::REMOTE }
+    let(:lfs_object) { create(:lfs_object, file_store: remote) }
+
+    context 'with object storage enabled' do
+      before do
+        stub_lfs_object_storage
+      end
+
+      it 'can store file remotely' do
+        allow(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async)
+
+        store_file(lfs_object)
+
+        expect(lfs_object.file_store).to eq remote
+        expect(lfs_object.file.path).not_to be_blank
+      end
+    end
+  end
+
+  def store_file(lfs_object)
+    lfs_object.file = fixture_file_upload(Rails.root.join("spec/fixtures/dk.png"), "`/png")
+    lfs_object.save!
+  end
 end