Add background migration for legacy artifacts

app/models/ci/job_artifact.rb

@@ -25,6 +25,11 @@ module Ci
       trace: 3
     }
+    enum path_type: {
+      era_2: nil,
+      era_1: 1
+    }
+
     def update_file_store
       # The file.object_store is set during `uploader.store!`
       # which happens after object is inserted/updated
@@ -61,7 +66,7 @@ module Ci
     end
     def update_project_statistics_after_destroy
-      update_project_statistics(-self.size)
+      update_project_statistics(-self.size) if self.size
     end
     def update_project_statistics(difference)

app/uploaders/job_artifact_uploader.rb

@@ -31,11 +31,15 @@ class JobArtifactUploader < GitlabUploader
     creation_date = model.created_at.utc.strftime('%Y_%m_%d')
-    File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
-              creation_date, model.job_id.to_s, model.id.to_s)
+    if model.era_2
+      File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
+                creation_date, model.job_id.to_s, model.id.to_s)
+    elsif model.era_1
+      File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s)
+    end
   end
   def disk_hash
-    @disk_hash ||= Digest::SHA2.hexdigest(model.project_id.to_s)
+    @disk_hash ||= Digest::SHA2.hexdigest(cproject_id.to_s)
   end
 end

db/migrate/20180427160409_add_legacy_path_to_ci_job_artifacts.rb

+class AddLegacyPathToCiJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_job_artifacts, :path_type, :integer
+  end
+end

db/post_migrate/20180427161409_migrate_legacy_artifacts_to_job_artifacts.rb

+class MigrateLegacyArtifactsToJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'MigrateLegacyArtifacts'.freeze
+  BATCH_SIZE = 500
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_builds'
+    self.inheritance_column = :_type_disabled # disable STI
+
+    scope :legacy_artifacts, -> { where('artifacts_file IS NOT NULL AND artifacts_file <> ?', '') }
+  end
+
+  def up
+    disable_statement_timeout
+
+    MigrateLegacyArtifactsToJobArtifacts::Build.legacy_artifacts.tap do |relation|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                            MIGRATION,
+                                                            5.minutes,
+                                                            batch_size: BATCH_SIZE)
+    end
+  end
+
+  def down
+    # There's nothing to revert for this migration.
+  end
+end

db/schema.rb

@@ -382,6 +382,7 @@ ActiveRecord::Schema.define(version: 20180603190921) do
     t.datetime_with_timezone "expire_at"
     t.string "file"
     t.binary "file_sha256"
+    t.integer "path_type"
   end
   add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree

lib/gitlab/background_migration/migrate_legacy_artifacts.rb

+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class MigrateLegacyArtifacts
+      class Build < ActiveRecord::Base
+        self.table_name = 'ci_builds'
+        self.inheritance_column = :_type_disabled
+
+        scope :legacy_artifacts, -> { where('artifacts_file IS NOT NULL AND artifacts_file <> ?', '') }
+      end
+
+      class JobArtifact < ActiveRecord::Base
+        self.table_name = 'ci_job_artifacts'
+
+        enum file_type: {
+          archive: 1,
+          metadata: 2,
+          trace: 3
+        }
+    
+        enum path_type: {
+          era_2: nil,
+          era_1: 1
+        }
+      end
+
+      def perform(start_id, stop_id)
+        rows = []
+
+        Gitlab::BackgroundMigration::MigrateLegacyArtifacts::Build.legacy_artifacts
+          .where(id: (start_id..stop_id))
+          .each do |build|
+            base_param = {
+              project_id: build.project_id,
+              job_id: build.id,
+              expire_at: build.artifacts_expire_at,
+              path_type: Gitlab::BackgroundMigration::MigrateLegacyArtifacts::JobArtifact.path_types['era_1'],
+              created_at: build.created_at,
+              updated_at: build.created_at
+            }
+
+            rows << base_param.merge({
+              size: build.artifacts_size,
+              file: build.artifacts_file,
+              file_store: build.artifacts_file_store,
+              file_type: Gitlab::BackgroundMigration::MigrateLegacyArtifacts::JobArtifact.file_types['archive'],
+              file_sha256: nil # `file_sha256` of legacy artifacts had not been persisted
+            })
+
+            if build.legacy_artifacts_metadata.exists?
+              rows << base_param.merge({
+                size: nil, # `size`` of legacy metadatas had not been persisted
+                file: build.artifacts_metadata,
+                file_store: build.artifacts_metadata_store,
+                file_type: Gitlab::BackgroundMigration::MigrateLegacyArtifacts::JobArtifact.file_types['metadata'],
+                file_sha256: nil # `file_sha256` of legacy artifacts had not been persisted
+              })
+            end
+          end
+
+          Gitlab::Database.bulk_insert(
+            Gitlab::BackgroundMigration::MigrateLegacyArtifacts::JobArtifact.table_name,
+            rows)
+      end
+    end
+  end
+end