Merge branch '40781-os-to-ce' into 'master'

Bring Object Storage to CE

Closes #4171, #4163, #3370, #2841, and #29203

See merge request gitlab-org/gitlab-ce!17358

lib/tasks/gitlab/artifacts/migrate.rake

+require 'logger'
+require 'resolv-replace'
+
+desc "GitLab | Migrate files for artifacts to comply with new storage format"
+namespace :gitlab do
+  namespace :artifacts do
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of artifacts')
+
+      Ci::Build.joins(:project)
+        .with_artifacts_stored_locally
+        .find_each(batch_size: 10) do |build|
+        begin
+          build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
+          build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
+
+          logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
+        rescue => e
+          logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+        end
+      end
+    end
+  end
+end

lib/tasks/gitlab/lfs/migrate.rake

+require 'logger'
+
+desc "GitLab | Migrate LFS objects to remote storage"
+namespace :gitlab do
+  namespace :lfs do
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of LFS files to object storage')
+
+      LfsObject.with_files_stored_locally
+        .find_each(batch_size: 10) do |lfs_object|
+          begin
+            lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+
+            logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
+          rescue => e
+            logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+          end
+        end
+    end
+  end
+end

lib/tasks/gitlab/uploads/migrate.rake

+namespace :gitlab do
+  namespace :uploads do
+    desc 'GitLab | Uploads | Migrate the uploaded files to object storage'
+    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
+      batch_size     = ENV.fetch('BATCH', 200).to_i
+      @to_store      = ObjectStorage::Store::REMOTE
+      @mounted_as    = args.mounted_as&.gsub(':', '')&.to_sym
+      @uploader_class = args.uploader_class.constantize
+      @model_class    = args.model_class.constantize
+
+      uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches
+    end
+
+    def enqueue_batch(batch, index)
+      job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
+                                                         @mounted_as,
+                                                         @to_store)
+      puts "Enqueued job ##{index}: #{job}"
+    rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
+      # continue for the next batch
+      puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
+    end
+
+    def uploads
+      Upload.class_eval { include EachBatch } unless Upload < EachBatch
+
+      Upload
+        .where.not(store: @to_store)
+        .where(uploader: @uploader_class.to_s,
+               model_type: @model_class.base_class.sti_name)
+    end
+  end
+end

spec/controllers/concerns/send_file_upload_spec.rb

+require 'spec_helper'
+
+describe SendFileUpload do
+  let(:uploader_class) do
+    Class.new(GitlabUploader) do
+      include ObjectStorage::Concern
+
+      storage_options Gitlab.config.uploads
+
+      private
+
+      # user/:id
+      def dynamic_segment
+        File.join(model.class.to_s.underscore, model.id.to_s)
+      end
+    end
+  end
+
+  let(:controller_class) do
+    Class.new do
+      include SendFileUpload
+    end
+  end
+
+  let(:object) { build_stubbed(:user) }
+  let(:uploader) { uploader_class.new(object, :file) }
+
+  describe '#send_upload' do
+    let(:controller) { controller_class.new }
+    let(:temp_file) { Tempfile.new('test') }
+
+    subject { controller.send_upload(uploader) }
+
+    before do
+      FileUtils.touch(temp_file)
+    end
+
+    after do
+      FileUtils.rm_f(temp_file)
+    end
+
+    context 'when local file is used' do
+      before do
+        uploader.store!(temp_file)
+      end
+
+      it 'sends a file' do
+        expect(controller).to receive(:send_file).with(uploader.path, anything)
+
+        subject
+      end
+    end
+
+    context 'when remote file is used' do
+      before do
+        stub_uploads_object_storage(uploader: uploader_class)
+        uploader.object_store = ObjectStorage::Store::REMOTE
+        uploader.store!(temp_file)
+      end
+
+      context 'and proxying is enabled' do
+        before do
+          allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { true }
+        end
+
+        it 'sends a file' do
+          headers = double
+          expect(headers).to receive(:store).with(Gitlab::Workhorse::SEND_DATA_HEADER, /^send-url:/)
+          expect(controller).to receive(:headers) { headers }
+          expect(controller).to receive(:head).with(:ok)
+
+          subject
+        end
+      end
+
+      context 'and proxying is disabled' do
+        before do
+          allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { false }
+        end
+
+        it 'sends a file' do
+          expect(controller).to receive(:redirect_to).with(/#{uploader.path}/)
+
+          subject
+        end
+      end
+    end
+  end
+end

spec/controllers/projects/artifacts_controller_spec.rb

@@ -145,9 +145,23 @@ describe Projects::ArtifactsController do
       context 'when using local file storage' do
         it_behaves_like 'a valid file' do
           let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) }
+          let(:store) { ObjectStorage::Store::LOCAL }
           let(:archive_path) { JobArtifactUploader.root }
         end
       end
+
+      context 'when using remote file storage' do
+        before do
+          stub_artifacts_object_storage
+        end
+
+        it_behaves_like 'a valid file' do
+          let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) }
+          let!(:job) { create(:ci_build, :success, pipeline: pipeline) }
+          let(:store) { ObjectStorage::Store::REMOTE }
+          let(:archive_path) { 'https://' }
+        end
+      end
     end
   end

spec/controllers/projects/jobs_controller_spec.rb

+# coding: utf-8
 require 'spec_helper'
 describe Projects::JobsController do
   include ApiHelpers
+  include HttpIOHelpers
   let(:project) { create(:project, :public) }
   let(:pipeline) { create(:ci_pipeline, project: project) }
@@ -203,6 +205,41 @@ describe Projects::JobsController do
       end
     end
+    context 'when trace artifact is in ObjectStorage' do
+      let!(:job) { create(:ci_build, :success, :trace_artifact, pipeline: pipeline) }
+
+      before do
+        allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+        allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url }
+        allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size }
+      end
+
+      context 'when there are no network issues' do
+        before do
+          stub_remote_trace_206
+
+          get_trace
+        end
+
+        it 'returns a trace' do
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(json_response['id']).to eq job.id
+          expect(json_response['status']).to eq job.status
+          expect(json_response['html']).to eq(job.trace.html)
+        end
+      end
+
+      context 'when there is a network issue' do
+        before do
+          stub_remote_trace_500
+        end
+
+        it 'returns a trace' do
+          expect { get_trace }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+        end
+      end
+    end
+
     def get_trace
       get :trace, namespace_id: project.namespace,
                   project_id: project,
@@ -446,14 +483,18 @@ describe Projects::JobsController do
   end
   describe 'GET raw' do
-    before do
-      get_raw
+    subject do
+      post :raw, namespace_id: project.namespace,
+                 project_id: project,
+                 id: job.id
     end
     context 'when job has a trace artifact' do
       let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
       it 'returns a trace' do
+        response = subject
+
         expect(response).to have_gitlab_http_status(:ok)
         expect(response.content_type).to eq 'text/plain; charset=utf-8'
         expect(response.body).to eq job.job_artifacts_trace.open.read
@@ -464,6 +505,8 @@ describe Projects::JobsController do
       let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) }
       it 'send a trace file' do
+        response = subject
+
         expect(response).to have_gitlab_http_status(:ok)
         expect(response.content_type).to eq 'text/plain; charset=utf-8'
         expect(response.body).to eq 'BUILD TRACE'
@@ -474,14 +517,22 @@ describe Projects::JobsController do
       let(:job) { create(:ci_build, pipeline: pipeline) }
       it 'returns not_found' do
+        response = subject
+
         expect(response).to have_gitlab_http_status(:not_found)
       end
     end
-    def get_raw
-      post :raw, namespace_id: project.namespace,
-                 project_id: project,
-                 id: job.id
+    context 'when the trace artifact is in ObjectStorage' do
+      let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+      before do
+        allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+      end
+
+      it 'redirect to the trace file url' do
+        expect(subject).to redirect_to(job.job_artifacts_trace.file.url)
+      end
     end
   end
 end

spec/controllers/projects/raw_controller_spec.rb

@@ -8,10 +8,7 @@ describe Projects::RawController do
       let(:id) { 'master/README.md' }
       it 'delivers ASCII file' do
-        get(:show,
-            namespace_id: public_project.namespace.to_param,
-            project_id: public_project,
-            id: id)
+        get_show(public_project, id)
         expect(response).to have_gitlab_http_status(200)
         expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8')
@@ -25,10 +22,7 @@ describe Projects::RawController do
       let(:id) { 'master/files/images/6049019_460s.jpg' }
       it 'sets image content type header' do
-        get(:show,
-            namespace_id: public_project.namespace.to_param,
-            project_id: public_project,
-            id: id)
+        get_show(public_project, id)
         expect(response).to have_gitlab_http_status(200)
         expect(response.header['Content-Type']).to eq('image/jpeg')
@@ -54,21 +48,40 @@ describe Projects::RawController do
           it 'serves the file' do
             expect(controller).to receive(:send_file).with("#{LfsObjectUploader.root}/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment')
-            get(:show,
-                namespace_id: public_project.namespace.to_param,
-                project_id: public_project,
-                id: id)
+            get_show(public_project, id)
             expect(response).to have_gitlab_http_status(200)
           end
+
+          context 'and lfs uses object storage' do
+            before do
+              lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png")
+              lfs_object.save!
+              stub_lfs_object_storage
+              lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+            end
+
+            it 'responds with redirect to file' do
+              get_show(public_project, id)
+
+              expect(response).to have_gitlab_http_status(302)
+              expect(response.location).to include(lfs_object.reload.file.path)
+            end
+
+            it 'sets content disposition' do
+              get_show(public_project, id)
+
+              file_uri = URI.parse(response.location)
+              params = CGI.parse(file_uri.query)
+
+              expect(params["response-content-disposition"].first).to eq 'attachment;filename="lfs_object.iso"'
+            end
+          end
         end
         context 'when project does not have access' do
           it 'does not serve the file' do
-            get(:show,
-                namespace_id: public_project.namespace.to_param,
-                project_id: public_project,
-                id: id)
+            get_show(public_project, id)
             expect(response).to have_gitlab_http_status(404)
           end
@@ -81,10 +94,7 @@ describe Projects::RawController do
         end
         it 'delivers ASCII file' do
-          get(:show,
-              namespace_id: public_project.namespace.to_param,
-              project_id: public_project,
-              id: id)
+          get_show(public_project, id)
           expect(response).to have_gitlab_http_status(200)
           expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8')
@@ -95,4 +105,10 @@ describe Projects::RawController do
       end
     end
   end
+
+  def get_show(project, id)
+    get(:show, namespace_id: project.namespace.to_param,
+               project_id: project,
+               id: id)
+  end
 end

spec/factories/appearances.rb

@@ -2,8 +2,21 @@
 FactoryBot.define do
   factory :appearance do
-    title       "MepMep"
-    description "This is my Community Edition instance"
+    title "GitLab Community Edition"
+    description "Open source software to collaborate on code"
     new_project_guidelines "Custom project guidelines"
   end
+
+  trait :with_logo do
+    logo { fixture_file_upload('spec/fixtures/dk.png') }
+  end
+
+  trait :with_header_logo do
+    header_logo { fixture_file_upload('spec/fixtures/dk.png') }
+  end
+
+  trait :with_logos do
+    with_logo
+    with_header_logo
+  end
 end

spec/factories/ci/job_artifacts.rb

@@ -5,6 +5,10 @@ FactoryBot.define do
     job factory: :ci_build
     file_type :archive
+    trait :remote_store do
+      file_store JobArtifactUploader::Store::REMOTE
+    end
+
     after :build do |artifact|
       artifact.project ||= artifact.job.project
     end

spec/factories/lfs_objects.rb

@@ -15,4 +15,8 @@ FactoryBot.define do
   trait :correct_oid do
     oid 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75'
   end
+
+  trait :object_storage do
+    file_store { LfsObjectUploader::Store::REMOTE }
+  end
 end

spec/factories/uploads.rb

@@ -5,6 +5,7 @@ FactoryBot.define do
     uploader "AvatarUploader"
     mount_point :avatar
     secret nil
+    store ObjectStorage::Store::LOCAL
     # we should build a mount agnostic upload by default
     transient do
@@ -27,6 +28,10 @@ FactoryBot.define do
       secret SecureRandom.hex
     end
+    trait :object_storage do
+      store ObjectStorage::Store::REMOTE
+    end
+
     trait :namespace_upload do
       model { build(:group) }
       path { File.join(secret, filename) }

spec/initializers/fog_google_https_private_urls_spec.rb

+require 'spec_helper'
+
+describe 'Fog::Storage::GoogleXML::File' do
+  let(:storage) do
+    Fog.mock!
+    Fog::Storage.new({
+                       google_storage_access_key_id: "asdf",
+                       google_storage_secret_access_key: "asdf",
+                       provider: "Google"
+                     })
+  end
+
+  let(:file) do
+    directory = storage.directories.create(key: 'data')
+    directory.files.create(
+      body: 'Hello World!',
+      key: 'hello_world.txt'
+    )
+  end
+
+  it 'delegates to #get_https_url' do
+    expect(file.url(Time.now)).to start_with("https://")
+  end
+end

spec/lib/gitlab/ci/trace/http_io_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Trace::HttpIO do
+  include HttpIOHelpers
+
+  let(:http_io) { described_class.new(url, size) }
+  let(:url) { remote_trace_url }
+  let(:size) { remote_trace_size }
+
+  describe '#close' do
+    subject { http_io.close }
+
+    it { is_expected.to be_nil }
+  end
+
+  describe '#binmode' do
+    subject { http_io.binmode }
+
+    it { is_expected.to be_nil }
+  end
+
+  describe '#binmode?' do
+    subject { http_io.binmode? }
+
+    it { is_expected.to be_truthy }
+  end
+
+  describe '#path' do
+    subject { http_io.path }
+
+    it { is_expected.to be_nil }
+  end
+
+  describe '#url' do
+    subject { http_io.url }
+
+    it { is_expected.to eq(url) }
+  end
+
+  describe '#seek' do
+    subject { http_io.seek(pos, where) }
+
+    context 'when moves pos to end of the file' do
+      let(:pos) { 0 }
+      let(:where) { IO::SEEK_END }
+
+      it { is_expected.to eq(size) }
+    end
+
+    context 'when moves pos to middle of the file' do
+      let(:pos) { size / 2 }
+      let(:where) { IO::SEEK_SET }
+
+      it { is_expected.to eq(size / 2) }
+    end
+
+    context 'when moves pos around' do
+      it 'matches the result' do
+        expect(http_io.seek(0)).to eq(0)
+        expect(http_io.seek(100, IO::SEEK_CUR)).to eq(100)
+        expect { http_io.seek(size + 1, IO::SEEK_CUR) }.to raise_error('new position is outside of file')
+      end
+    end
+  end
+
+  describe '#eof?' do
+    subject { http_io.eof? }
+
+    context 'when current pos is at end of the file' do
+      before do
+        http_io.seek(size, IO::SEEK_SET)
+      end
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when current pos is not at end of the file' do
+      before do
+        http_io.seek(0, IO::SEEK_SET)
+      end
+
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  describe '#each_line' do
+    subject { http_io.each_line }
+
+    let(:string_io) { StringIO.new(remote_trace_body) }
+
+    before do
+      stub_remote_trace_206
+    end
+
+    it 'yields lines' do
+      expect { |b| http_io.each_line(&b) }.to yield_successive_args(*string_io.each_line.to_a)
+    end
+
+    context 'when buckets on GCS' do
+      context 'when BUFFER_SIZE is larger than file size' do
+        before do
+          stub_remote_trace_200
+          set_larger_buffer_size_than(size)
+        end
+
+        it 'calls get_chunk only once' do
+          expect_any_instance_of(Net::HTTP).to receive(:request).once.and_call_original
+
+          http_io.each_line { |line| }
+        end
+      end
+    end
+  end
+
+  describe '#read' do
+    subject { http_io.read(length) }
+
+    context 'when there are no network issue' do
+      before do
+        stub_remote_trace_206
+      end
+
+      context 'when read whole size' do
+        let(:length) { nil }
+
+        context 'when BUFFER_SIZE is smaller than file size' do
+          before do
+            set_smaller_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body)
+          end
+        end
+
+        context 'when BUFFER_SIZE is larger than file size' do
+          before do
+            set_larger_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body)
+          end
+        end
+      end
+
+      context 'when read only first 100 bytes' do
+        let(:length) { 100 }
+
+        context 'when BUFFER_SIZE is smaller than file size' do
+          before do
+            set_smaller_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body[0, length])
+          end
+        end
+
+        context 'when BUFFER_SIZE is larger than file size' do
+          before do
+            set_larger_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body[0, length])
+          end
+        end
+      end
+
+      context 'when tries to read oversize' do
+        let(:length) { size + 1000 }
+
+        context 'when BUFFER_SIZE is smaller than file size' do
+          before do
+            set_smaller_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body)
+          end
+        end
+
+        context 'when BUFFER_SIZE is larger than file size' do
+          before do
+            set_larger_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body)
+          end
+        end
+      end
+
+      context 'when tries to read 0 bytes' do
+        let(:length) { 0 }
+
+        context 'when BUFFER_SIZE is smaller than file size' do
+          before do
+            set_smaller_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to be_empty
+          end
+        end
+
+        context 'when BUFFER_SIZE is larger than file size' do
+          before do
+            set_larger_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to be_empty
+          end
+        end
+      end
+    end
+
+    context 'when there is anetwork issue' do
+      let(:length) { nil }
+
+      before do
+        stub_remote_trace_500
+      end
+
+      it 'reads a trace' do
+        expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+      end
+    end
+  end
+
+  describe '#readline' do
+    subject { http_io.readline }
+
+    let(:string_io) { StringIO.new(remote_trace_body) }
+
+    before do
+      stub_remote_trace_206
+    end
+
+    shared_examples 'all line matching' do
+      it 'reads a line' do
+        (0...remote_trace_body.lines.count).each do
+          expect(http_io.readline).to eq(string_io.readline)
+        end
+      end
+    end
+
+    context 'when there is anetwork issue' do
+      let(:length) { nil }
+
+      before do
+        stub_remote_trace_500
+      end
+
+      it 'reads a trace' do
+        expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+      end
+    end
+
+    context 'when BUFFER_SIZE is smaller than file size' do
+      before do
+        set_smaller_buffer_size_than(size)
+      end
+
+      it_behaves_like 'all line matching'
+    end
+
+    context 'when BUFFER_SIZE is larger than file size' do
+      before do
+        set_larger_buffer_size_than(size)
+      end
+
+      it_behaves_like 'all line matching'
+    end
+
+    context 'when pos is at middle of the file' do
+      before do
+        set_smaller_buffer_size_than(size)
+
+        http_io.seek(size / 2)
+        string_io.seek(size / 2)
+      end
+
+      it 'reads from pos' do
+        expect(http_io.readline).to eq(string_io.readline)
+      end
+    end
+  end
+
+  describe '#write' do
+    subject { http_io.write(nil) }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+
+  describe '#truncate' do
+    subject { http_io.truncate(nil) }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+
+  describe '#flush' do
+    subject { http_io.flush }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+
+  describe '#present?' do
+    subject { http_io.present? }
+
+    it { is_expected.to be_truthy }
+  end
+end

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -265,7 +265,9 @@ CommitStatus:
 - target_url
 - description
 - artifacts_file
+- artifacts_file_store
 - artifacts_metadata
+- artifacts_metadata_store
 - erased_by_id
 - erased_at
 - artifacts_expire_at

spec/lib/gitlab/verify/lfs_objects_spec.rb

@@ -31,5 +31,21 @@ describe Gitlab::Verify::LfsObjects do
       expect(failures.keys).to contain_exactly(lfs_object)
       expect(failure.to_s).to include('Checksum mismatch')
     end
+
+    context 'with remote files' do
+      before do
+        stub_lfs_object_storage
+      end
+
+      it 'skips LFS objects in object storage' do
+        local_failure = create(:lfs_object)
+        create(:lfs_object, :object_storage)
+
+        failures = {}
+        described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) }
+
+        expect(failures.keys).to contain_exactly(local_failure)
+      end
+    end
   end
 end

spec/lib/gitlab/verify/uploads_spec.rb

@@ -40,5 +40,21 @@ describe Gitlab::Verify::Uploads do
       expect(failures.keys).to contain_exactly(upload)
       expect(failure.to_s).to include('Checksum missing')
     end
+
+    context 'with remote files' do
+      before do
+        stub_uploads_object_storage(AvatarUploader)
+      end
+
+      it 'skips uploads in object storage' do
+        local_failure = create(:upload)
+        create(:upload, :object_storage)
+
+        failures = {}
+        described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) }
+
+        expect(failures.keys).to contain_exactly(local_failure)
+      end
+    end
   end
 end

spec/migrations/remove_empty_fork_networks_spec.rb

@@ -19,6 +19,10 @@ describe RemoveEmptyForkNetworks, :migration do
     deleted_project.destroy!
   end
+  after do
+    Upload.reset_column_information
+  end
+
   it 'deletes only the fork network without members' do
     expect(fork_networks.count).to eq(2)

spec/models/ci/build_spec.rb

@@ -198,6 +198,16 @@ describe Ci::Build do
     end
     context 'when legacy artifacts are used' do
+      let(:build) { create(:ci_build, :legacy_artifacts) }
+
+      subject { build.artifacts? }
+
+      context 'is expired' do
+        let(:build) { create(:ci_build, :legacy_artifacts, :expired) }
+
+        it { is_expected.to be_falsy }
+      end
+
       context 'artifacts archive does not exist' do
         let(:build) { create(:ci_build) }
@@ -208,13 +218,25 @@ describe Ci::Build do
         let(:build) { create(:ci_build, :legacy_artifacts) }
         it { is_expected.to be_truthy }
-        context 'is expired' do
-          let(:build) { create(:ci_build, :legacy_artifacts, :expired) }
-          it { is_expected.to be_falsy }
-        end
+      end
+    end
+  end
+  describe '#browsable_artifacts?' do
+    subject { build.browsable_artifacts? }
+    context 'artifacts metadata does not exist' do
+      before do
+        build.update_attributes(legacy_artifacts_metadata: nil)
       end
+
+      it { is_expected.to be_falsy }
+    end
+
+    context 'artifacts metadata does exists' do
+      let(:build) { create(:ci_build, :artifacts) }
+
+      it { is_expected.to be_truthy }
     end
   end

spec/models/ci/job_artifact_spec.rb

@@ -15,6 +15,50 @@ describe Ci::JobArtifact do
   it { is_expected.to delegate_method(:open).to(:file) }
   it { is_expected.to delegate_method(:exists?).to(:file) }
+  describe 'callbacks' do
+    subject { create(:ci_job_artifact, :archive) }
+
+    describe '#schedule_background_upload' do
+      context 'when object storage is disabled' do
+        before do
+          stub_artifacts_object_storage(enabled: false)
+        end
+
+        it 'does not schedule the migration' do
+          expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
+
+          subject
+        end
+      end
+
+      context 'when object storage is enabled' do
+        context 'when background upload is enabled' do
+          before do
+            stub_artifacts_object_storage(background_upload: true)
+          end
+
+          it 'schedules the model for migration' do
+            expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('JobArtifactUploader', described_class.name, :file, kind_of(Numeric))
+
+            subject
+          end
+        end
+
+        context 'when background upload is disabled' do
+          before do
+            stub_artifacts_object_storage(background_upload: false)
+          end
+
+          it 'schedules the model for migration' do
+            expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+            subject
+          end
+        end
+      end
+    end
+  end
+
   describe '#set_size' do
     it 'sets the size' do
       expect(artifact.size).to eq(106365)

spec/models/lfs_object_spec.rb

+require 'spec_helper'
+
+describe LfsObject do
+  describe '#local_store?' do
+    it 'returns true when file_store is nil' do
+      subject.file_store = nil
+
+      expect(subject.local_store?).to eq true
+    end
+
+    it 'returns true when file_store is equal to LfsObjectUploader::Store::LOCAL' do
+      subject.file_store = LfsObjectUploader::Store::LOCAL
+
+      expect(subject.local_store?).to eq true
+    end
+
+    it 'returns false whe file_store is equal to LfsObjectUploader::Store::REMOTE' do
+      subject.file_store = LfsObjectUploader::Store::REMOTE
+
+      expect(subject.local_store?).to eq false
+    end
+  end
+
+  describe '#schedule_background_upload' do
+    before do
+      stub_lfs_setting(enabled: true)
+    end
+
+    subject { create(:lfs_object, :with_file) }
+
+    context 'when object storage is disabled' do
+      before do
+        stub_lfs_object_storage(enabled: false)
+      end
+
+      it 'does not schedule the migration' do
+        expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+        subject
+      end
+    end
+
+    context 'when object storage is enabled' do
+      context 'when background upload is enabled' do
+        context 'when is licensed' do
+          before do
+            stub_lfs_object_storage(background_upload: true)
+          end
+
+          it 'schedules the model for migration' do
+            expect(ObjectStorage::BackgroundMoveWorker)
+              .to receive(:perform_async)
+              .with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric))
+              .once
+
+            subject
+          end
+
+          it 'schedules the model for migration once' do
+            expect(ObjectStorage::BackgroundMoveWorker)
+              .to receive(:perform_async)
+              .with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric))
+              .once
+
+            lfs_object = create(:lfs_object)
+            lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png")
+            lfs_object.save!
+          end
+        end
+      end
+
+      context 'when background upload is disabled' do
+        before do
+          stub_lfs_object_storage(background_upload: false)
+        end
+
+        it 'schedules the model for migration' do
+          expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+          subject
+        end
+      end
+    end
+  end
+end