Add pipeline stages post deployment migration

db/migrate/20170525132202_migrate_pipeline_stages.rb → db/migrate/20170525132202_create_pipeline_stages.rb

-class MigratePipelineStages < ActiveRecord::Migration
+class CreatePipelineStages < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
   DOWNTIME = false

db/post_migrate/20170526101042_migrate_pipeline_stages.rb

+class MigratePipelineStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    disable_statement_timeout
+
+    execute <<-SQL.strip_heredoc
+      INSERT INTO ci_stages (project_id, pipeline_id, name)
+        SELECT project_id, commit_id, stage FROM ci_builds
+          WHERE stage IS NOT NULL
+          GROUP BY project_id, commit_id, stage, stage_idx
+          ORDER BY stage_idx
+    SQL
+
+    add_concurrent_index(:ci_stages, [:pipeline_id, :name])
+
+    add_column(:ci_builds, :stage_id, :integer)
+
+    stage_id = Arel.sql('(SELECT id FROM ci_stages ' \
+                         'WHERE ci_stages.pipeline_id = ci_builds.commit_id ' \
+                         'AND ci_stages.name = ci_builds.stage)')
+    update_column_in_batches(:ci_builds, :stage_id, stage_id)
+
+    # add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade
+    # add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade
+  end
+
+  def down
+    execute('TRUNCATE TABLE ci_stages')
+
+    if column_exists?(:ci_builds, :stage_id)
+      remove_column(:ci_builds, :stage_id)
+    end
+
+    if index_exists?(:ci_stages, [:pipeline_id, :name])
+      remove_index(:ci_stages, [:pipeline_id, :name])
+    end
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20170525132202) do
+ActiveRecord::Schema.define(version: 20170526101042) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -233,6 +233,7 @@ ActiveRecord::Schema.define(version: 20170525132202) do
     t.string "coverage_regex"
     t.integer "auto_canceled_by_id"
     t.boolean "retried"
+    t.integer "stage_id"
   end
   add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
@@ -333,6 +334,8 @@ ActiveRecord::Schema.define(version: 20170525132202) do
     t.datetime "updated_at"
   end
+  add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree
+
   create_table "ci_trigger_requests", force: :cascade do |t|
     t.integer "trigger_id", null: false
     t.text "variables"

spec/migrations/migrate_pipeline_stages_spec.rb

 require 'spec_helper'
-require Rails.root.join('db', 'migrate', '20170525132202_migrate_pipeline_stages.rb')
-describe MigratePipelineStages, :migration, schema: 20170523091700 do
+require Rails.root.join('db', 'post_migrate', '20170526101042_migrate_pipeline_stages.rb')
+describe MigratePipelineStages, :migration, schema: 20170525132202 do
   ##
-  # Create test data
+  # Create test data - pipeline and CI/CD jobs.
   #
+
+  let(:jobs) { table(:ci_builds) }
+  let(:stages) { table(:ci_stages) }
+  let(:pipelines) { table(:ci_pipelines) }
+
   before do
-    table(:ci_pipelines).create!(ref: 'master', sha: 'adf43c3a')
+    # Create CI/CD pipelines
+    #
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+    pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb')
+
+    # Create CI/CD jobs
+    #
+    jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
+    jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
+    jobs.create!(id: 5, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
+    jobs.create!(id: 8, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 9, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 10, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
+    jobs.create!(id: 11, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy')
   end
-  it 'correctly migrates pipeline stages' do |migration, meta|
-    expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq false
+  it 'correctly migrates pipeline stages' do
+    expect(stages.count).to be_zero
     migrate!
-    expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq true
+    expect(stages.count).to eq 7
+    expect(stages.all.pluck(:name))
+      .to match_array %w[test build deploy test:1 test:2 deploy deploy]
+    expect(stages.where(pipeline_id: 1).order(:id).pluck(:name))
+      .to eq %w[test build deploy]
+    expect(stages.where(pipeline_id: 2).order(:id).pluck(:name))
+      .to eq %w[test:1 test:2 deploy]
+
+    expect(jobs.where(stage_id: nil)).to be_empty
   end
 end

spec/spec_helper.rb

@@ -95,8 +95,8 @@ RSpec.configure do |config|
   end
   config.around(:example, migration: true) do |example|
-    ActiveRecord::Migrator
-      .migrate(migrations_paths, example.metadata.fetch(:schema))
+    schema_version = example.metadata.fetch(:schema)
+    ActiveRecord::Migrator.migrate(migrations_paths, schema_version)
     example.run

spec/support/db_cleaner.rb

@@ -19,6 +19,10 @@ RSpec.configure do |config|
     DatabaseCleaner.strategy = :truncation
   end
+  config.before(:each, :migration) do
+    DatabaseCleaner.strategy = :truncation
+  end
+
   config.before(:each) do
     DatabaseCleaner.start
   end

spec/support/migrations_helpers.rb

@@ -7,6 +7,10 @@ module MigrationsHelpers
     ActiveRecord::Migrator.migrations_paths
   end
+  def table_exists?(name)
+    ActiveRecord::Base.connection.table_exists?(name)
+  end
+
   def migrate!
     ActiveRecord::Migrator.up(migrations_paths) do |migration|
       migration.name == described_class.name