Merge branch '40781-os-to-ce' into 'master'

Bring Object Storage to CE

Closes #4171, #4163, #3370, #2841, and #29203

See merge request gitlab-org/gitlab-ce!17358

spec/uploaders/namespace_file_uploader_spec.rb

@@ -13,4 +13,26 @@ describe NamespaceFileUploader do
                   store_dir: %r[uploads/-/system/namespace/\d+],
                   upload_path: IDENTIFIER,
                   absolute_path: %r[#{CarrierWave.root}/uploads/-/system/namespace/\d+/#{IDENTIFIER}]
+
+  context "object_store is REMOTE" do
+    before do
+      stub_uploads_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like 'builds correct paths',
+                    store_dir: %r[namespace/\d+/\h+],
+                    upload_path: IDENTIFIER
+  end
+
+  describe "#migrate!" do
+    before do
+      uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt')))
+      stub_uploads_object_storage
+    end
+
+    it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+    it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+  end
 end

spec/uploaders/object_storage_spec.rb

+require 'rails_helper'
+require 'carrierwave/storage/fog'
+
+class Implementation < GitlabUploader
+  include ObjectStorage::Concern
+  include ::RecordsUploads::Concern
+  prepend ::ObjectStorage::Extension::RecordsUploads
+
+  storage_options Gitlab.config.uploads
+
+  private
+
+  # user/:id
+  def dynamic_segment
+    File.join(model.class.to_s.underscore, model.id.to_s)
+  end
+end
+
+describe ObjectStorage do
+  let(:uploader_class) { Implementation }
+  let(:object) { build_stubbed(:user) }
+  let(:uploader) { uploader_class.new(object, :file) }
+
+  before do
+    allow(uploader_class).to receive(:object_store_enabled?).and_return(true)
+  end
+
+  describe '#object_store=' do
+    it "reload the local storage" do
+      uploader.object_store = described_class::Store::LOCAL
+      expect(uploader.file_storage?).to be_truthy
+    end
+
+    it "reload the REMOTE storage" do
+      uploader.object_store = described_class::Store::REMOTE
+      expect(uploader.file_storage?).to be_falsey
+    end
+  end
+
+  context 'object_store is Store::LOCAL' do
+    before do
+      uploader.object_store = described_class::Store::LOCAL
+    end
+
+    describe '#store_dir' do
+      it 'is the composition of (base_dir, dynamic_segment)' do
+        expect(uploader.store_dir).to start_with("uploads/-/system/user/")
+      end
+    end
+  end
+
+  context 'object_store is Store::REMOTE' do
+    before do
+      uploader.object_store = described_class::Store::REMOTE
+    end
+
+    describe '#store_dir' do
+      it 'is the composition of (dynamic_segment)' do
+        expect(uploader.store_dir).to start_with("user/")
+      end
+    end
+  end
+
+  describe '#object_store' do
+    it "delegates to <mount>_store on model" do
+      expect(object).to receive(:file_store)
+
+      uploader.object_store
+    end
+
+    context 'when store is null' do
+      before do
+        expect(object).to receive(:file_store).and_return(nil)
+      end
+
+      it "returns Store::LOCAL" do
+        expect(uploader.object_store).to eq(described_class::Store::LOCAL)
+      end
+    end
+
+    context 'when value is set' do
+      before do
+        expect(object).to receive(:file_store).and_return(described_class::Store::REMOTE)
+      end
+
+      it "returns the given value" do
+        expect(uploader.object_store).to eq(described_class::Store::REMOTE)
+      end
+    end
+  end
+
+  describe '#file_cache_storage?' do
+    context 'when file storage is used' do
+      before do
+        uploader_class.cache_storage(:file)
+      end
+
+      it { expect(uploader).to be_file_cache_storage }
+    end
+
+    context 'when is remote storage' do
+      before do
+        uploader_class.cache_storage(:fog)
+      end
+
+      it { expect(uploader).not_to be_file_cache_storage }
+    end
+  end
+
+  # this means the model shall include
+  #   include RecordsUpload::Concern
+  #   prepend ObjectStorage::Extension::RecordsUploads
+  # the object_store persistence is delegated to the `Upload` model.
+  #
+  context 'when persist_object_store? is false' do
+    let(:object) { create(:project, :with_avatar) }
+    let(:uploader) { object.avatar }
+
+    it { expect(object).to be_a(Avatarable) }
+    it { expect(uploader.persist_object_store?).to be_falsey }
+
+    describe 'delegates the object_store logic to the `Upload` model' do
+      it 'sets @upload to the found `upload`' do
+        expect(uploader.upload).to eq(uploader.upload)
+      end
+
+      it 'sets @object_store to the `Upload` value' do
+        expect(uploader.object_store).to eq(uploader.upload.store)
+      end
+    end
+
+    describe '#migrate!' do
+      let(:new_store) { ObjectStorage::Store::REMOTE }
+
+      before do
+        stub_uploads_object_storage(uploader: AvatarUploader)
+      end
+
+      subject { uploader.migrate!(new_store) }
+
+      it 'persist @object_store to the recorded upload' do
+        subject
+
+        expect(uploader.upload.store).to eq(new_store)
+      end
+
+      describe 'fails' do
+        it 'is handled gracefully' do
+          store = uploader.object_store
+          expect_any_instance_of(Upload).to receive(:save!).and_raise("An error")
+
+          expect { subject }.to raise_error("An error")
+          expect(uploader.exists?).to be_truthy
+          expect(uploader.upload.store).to eq(store)
+        end
+      end
+    end
+  end
+
+  # this means the model holds an <mounted_as>_store attribute directly
+  # and do not delegate the object_store persistence to the `Upload` model.
+  #
+  context 'persist_object_store? is true' do
+    context 'when using JobArtifactsUploader' do
+      let(:store) { described_class::Store::LOCAL }
+      let(:object) { create(:ci_job_artifact, :archive, file_store: store) }
+      let(:uploader) { object.file }
+
+      context 'checking described_class' do
+        it "uploader include described_class::Concern" do
+          expect(uploader).to be_a(described_class::Concern)
+        end
+      end
+
+      describe '#use_file' do
+        context 'when file is stored locally' do
+          it "calls a regular path" do
+            expect { |b| uploader.use_file(&b) }.not_to yield_with_args(%r[tmp/cache])
+          end
+        end
+
+        context 'when file is stored remotely' do
+          let(:store) { described_class::Store::REMOTE }
+
+          before do
+            stub_artifacts_object_storage
+          end
+
+          it "calls a cache path" do
+            expect { |b| uploader.use_file(&b) }.to yield_with_args(%r[tmp/cache])
+          end
+        end
+      end
+
+      describe '#migrate!' do
+        subject { uploader.migrate!(new_store) }
+
+        shared_examples "updates the underlying <mounted>_store" do
+          it do
+            subject
+
+            expect(object.file_store).to eq(new_store)
+          end
+        end
+
+        context 'when using the same storage' do
+          let(:new_store) { store }
+
+          it "to not migrate the storage" do
+            subject
+
+            expect(uploader).not_to receive(:store!)
+            expect(uploader.object_store).to eq(store)
+          end
+        end
+
+        context 'when migrating to local storage' do
+          let(:store) { described_class::Store::REMOTE }
+          let(:new_store) { described_class::Store::LOCAL }
+
+          before do
+            stub_artifacts_object_storage
+          end
+
+          include_examples "updates the underlying <mounted>_store"
+
+          it "local file does not exist" do
+            expect(File.exist?(uploader.path)).to eq(false)
+          end
+
+          it "remote file exist" do
+            expect(uploader.file.exists?).to be_truthy
+          end
+
+          it "does migrate the file" do
+            subject
+
+            expect(uploader.object_store).to eq(new_store)
+            expect(File.exist?(uploader.path)).to eq(true)
+          end
+        end
+
+        context 'when migrating to remote storage' do
+          let(:new_store) { described_class::Store::REMOTE }
+          let!(:current_path) { uploader.path }
+
+          it "file does exist" do
+            expect(File.exist?(current_path)).to eq(true)
+          end
+
+          context 'when storage is disabled' do
+            before do
+              stub_artifacts_object_storage(enabled: false)
+            end
+
+            it "to raise an error" do
+              expect { subject }.to raise_error(/Object Storage is not enabled/)
+            end
+          end
+
+          context 'when credentials are set' do
+            before do
+              stub_artifacts_object_storage
+            end
+
+            include_examples "updates the underlying <mounted>_store"
+
+            it "does migrate the file" do
+              subject
+
+              expect(uploader.object_store).to eq(new_store)
+            end
+
+            it "does delete original file" do
+              subject
+
+              expect(File.exist?(current_path)).to eq(false)
+            end
+
+            context 'when subject save fails' do
+              before do
+                expect(uploader).to receive(:persist_object_store!).and_raise(RuntimeError, "exception")
+              end
+
+              it "original file is not removed" do
+                expect { subject }.to raise_error(/exception/)
+
+                expect(File.exist?(current_path)).to eq(true)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  describe '#fog_directory' do
+    let(:remote_directory) { 'directory' }
+
+    before do
+      uploader_class.storage_options double(object_store: double(remote_directory: remote_directory))
+    end
+
+    subject { uploader.fog_directory }
+
+    it { is_expected.to eq(remote_directory) }
+  end
+
+  describe '#fog_credentials' do
+    let(:connection) { Settingslogic.new("provider" => "AWS") }
+
+    before do
+      uploader_class.storage_options double(object_store: double(connection: connection))
+    end
+
+    subject { uploader.fog_credentials }
+
+    it { is_expected.to eq(provider: 'AWS') }
+  end
+
+  describe '#fog_public' do
+    subject { uploader.fog_public }
+
+    it { is_expected.to eq(false) }
+  end
+end

spec/uploaders/personal_file_uploader_spec.rb

@@ -14,6 +14,18 @@ describe PersonalFileUploader do
                   upload_path: IDENTIFIER,
                   absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet/\d+/#{IDENTIFIER}]
+  context "object_store is REMOTE" do
+    before do
+      stub_uploads_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like 'builds correct paths',
+                    store_dir: %r[\d+/\h+],
+                    upload_path: IDENTIFIER
+  end
+
   describe '#to_h' do
     before do
       subject.instance_variable_set(:@secret, 'secret')
@@ -30,4 +42,14 @@ describe PersonalFileUploader do
       )
     end
   end
+
+  describe "#migrate!" do
+    before do
+      uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt')))
+      stub_uploads_object_storage
+    end
+
+    it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+    it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+  end
 end

spec/workers/object_storage_upload_worker_spec.rb

+require 'spec_helper'
+
+describe ObjectStorageUploadWorker do
+  let(:local) { ObjectStorage::Store::LOCAL }
+  let(:remote) { ObjectStorage::Store::REMOTE }
+
+  def perform
+    described_class.perform_async(uploader_class.name, subject_class, file_field, subject_id)
+  end
+
+  context 'for LFS' do
+    let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) }
+    let(:uploader_class) { LfsObjectUploader }
+    let(:subject_class) { LfsObject }
+    let(:file_field) { :file }
+    let(:subject_id) { lfs_object.id }
+
+    context 'when object storage is enabled' do
+      before do
+        stub_lfs_object_storage(background_upload: true)
+      end
+
+      it 'uploads object to storage' do
+        expect { perform }.to change { lfs_object.reload.file_store }.from(local).to(remote)
+      end
+
+      context 'when background upload is disabled' do
+        before do
+          allow(Gitlab.config.lfs.object_store).to receive(:background_upload) { false }
+        end
+
+        it 'is skipped' do
+          expect { perform }.not_to change { lfs_object.reload.file_store }
+        end
+      end
+    end
+
+    context 'when object storage is disabled' do
+      before do
+        stub_lfs_object_storage(enabled: false)
+      end
+
+      it "doesn't migrate files" do
+        perform
+
+        expect(lfs_object.reload.file_store).to eq(local)
+      end
+    end
+  end
+
+  context 'for legacy artifacts' do
+    let(:build) { create(:ci_build, :legacy_artifacts) }
+    let(:uploader_class) { LegacyArtifactUploader }
+    let(:subject_class) { Ci::Build }
+    let(:file_field) { :artifacts_file }
+    let(:subject_id) { build.id }
+
+    context 'when local storage is used' do
+      let(:store) { local }
+
+      context 'and remote storage is defined' do
+        before do
+          stub_artifacts_object_storage(background_upload: true)
+        end
+
+        it "migrates file to remote storage" do
+          perform
+
+          expect(build.reload.artifacts_file_store).to eq(remote)
+        end
+
+        context 'for artifacts_metadata' do
+          let(:file_field) { :artifacts_metadata }
+
+          it 'migrates metadata to remote storage' do
+            perform
+
+            expect(build.reload.artifacts_metadata_store).to eq(remote)
+          end
+        end
+      end
+    end
+  end
+
+  context 'for job artifacts' do
+    let(:artifact) { create(:ci_job_artifact, :archive) }
+    let(:uploader_class) { JobArtifactUploader }
+    let(:subject_class) { Ci::JobArtifact }
+    let(:file_field) { :file }
+    let(:subject_id) { artifact.id }
+
+    context 'when local storage is used' do
+      let(:store) { local }
+
+      context 'and remote storage is defined' do
+        before do
+          stub_artifacts_object_storage(background_upload: true)
+        end
+
+        it "migrates file to remote storage" do
+          perform
+
+          expect(artifact.reload.file_store).to eq(remote)
+        end
+      end
+    end
+  end
+end