Merge branch 'merge-schema-backport-to-auto-deploy' into '12-1-auto-deploy-0010836'

Merge database schema backport into the auto deploy branch

See merge request gitlab-org/gitlab-ce!29773

.gitlab/ci/rails.gitlab-ci.yml

@@ -111,7 +111,7 @@
   variables:
     SETUP_DB: "false"
   script:
-    - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0
+    - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0
     - git checkout -f FETCH_HEAD
     - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile
     - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile

changelogs/unreleased/backport-schema-changes.yml

+---
+title: Backport the EE schema and migrations to CE
+merge_request: 26940
+author: Yorick Peterse
+type: other

danger/database/Dangerfile

@@ -13,7 +13,6 @@ DB_FILES = [
   'lib/gitlab/github_import/',
   'lib/gitlab/sql/',
   'rubocop/cop/migration',
-  'ee/db/',
   'ee/lib/gitlab/database/'
 ].freeze

db/migrate/20161007073613_create_user_activities.rb

+
 class CreateUserActivities < ActiveRecord::Migration[4.2]
-  DOWNTIME = false
-  # This migration is a no-op. It just exists to match EE.
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = true
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  DOWNTIME_REASON = 'Adding foreign key'.freeze
+
+  # When using the methods "add_concurrent_index" or "add_column_with_default"
+  # you must disable the use of transactions as these methods can not run in an
+  # existing transaction. When using "add_concurrent_index" make sure that this
+  # method is the _only_ method called in the migration, any other changes
+  # should go in a separate migration. This ensures that upon failure _only_ the
+  # index creation fails and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+  # disable_ddl_transaction!
   def change
+    create_table :user_activities do |t|
+      t.belongs_to :user, index: { unique: true }, foreign_key: { on_delete: :cascade }
+      t.datetime :last_activity_at, null: false
+    end
   end
 end

db/migrate/20161117114805_remove_undeleted_groups.rb

-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
-class RemoveUndeletedGroups < ActiveRecord::Migration[4.2]
-  DOWNTIME = false
-
-  def up
-    is_ee = defined?(Gitlab::License)
-
-    if is_ee
-      execute <<-EOF.strip_heredoc
-      DELETE FROM path_locks
-      WHERE project_id IN (
-        SELECT project_id
-        FROM projects
-        WHERE namespace_id IN (#{namespaces_pending_removal})
-      );
-      EOF
-
-      execute <<-EOF.strip_heredoc
-      DELETE FROM remote_mirrors
-      WHERE project_id IN (
-        SELECT project_id
-        FROM projects
-        WHERE namespace_id IN (#{namespaces_pending_removal})
-      );
-      EOF
-    end
-
-    execute <<-EOF.strip_heredoc
-    DELETE FROM lists
-    WHERE label_id IN (
-      SELECT id
-      FROM labels
-      WHERE group_id IN (#{namespaces_pending_removal})
-    );
-    EOF
-
-    execute <<-EOF.strip_heredoc
-    DELETE FROM lists
-    WHERE board_id IN (
-      SELECT id
-      FROM boards
-      WHERE project_id IN (
-        SELECT project_id
-        FROM projects
-        WHERE namespace_id IN (#{namespaces_pending_removal})
-      )
-    );
-    EOF
-
-    execute <<-EOF.strip_heredoc
-    DELETE FROM labels
-    WHERE group_id IN (#{namespaces_pending_removal});
-    EOF
-
-    execute <<-EOF.strip_heredoc
-    DELETE FROM boards
-    WHERE project_id IN (
-      SELECT project_id
-      FROM projects
-      WHERE namespace_id IN (#{namespaces_pending_removal})
-    )
-    EOF
-
-    execute <<-EOF.strip_heredoc
-    DELETE FROM projects
-    WHERE namespace_id IN (#{namespaces_pending_removal});
-    EOF
-
-    if is_ee
-      # EE adds these columns but we have to make sure this data is cleaned up
-      # here before we run the DELETE below. An alternative would be patching
-      # this migration in EE but this will only result in a mess and confusing
-      # migrations.
-      execute <<-EOF.strip_heredoc
-      DELETE FROM protected_branch_push_access_levels
-      WHERE group_id IN (#{namespaces_pending_removal});
-      EOF
-
-      execute <<-EOF.strip_heredoc
-      DELETE FROM protected_branch_merge_access_levels
-      WHERE group_id IN (#{namespaces_pending_removal});
-      EOF
-    end
-
-    # This removes namespaces that were supposed to be deleted but still reside
-    # in the database.
-    execute "DELETE FROM namespaces WHERE deleted_at IS NOT NULL;"
-  end
-
-  def down
-    # This is an irreversible migration;
-    # If someone is trying to rollback for other reasons, we should not throw an Exception.
-    # raise ActiveRecord::IrreversibleMigration
-  end
-
-  def namespaces_pending_removal
-    "SELECT id FROM (
-      SELECT id
-      FROM namespaces
-      WHERE deleted_at IS NOT NULL
-    ) namespace_ids"
-  end
-end

db/migrate/20171121135738_clean_up_from_merge_request_diffs_and_commits.rb

-class CleanUpFromMergeRequestDiffsAndCommits < ActiveRecord::Migration[4.2]
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-
-  class MergeRequestDiff < ActiveRecord::Base
-    self.table_name = 'merge_request_diffs'
-
-    include ::EachBatch
-  end
-
-  disable_ddl_transaction!
-
-  def up
-    Gitlab::BackgroundMigration.steal('DeserializeMergeRequestDiffsAndCommits')
-
-    # The literal '--- []\n' value is created by the import process and treated
-    # as null by the application, so we can ignore those - even if we were
-    # migrating, it wouldn't create any rows.
-    literal_prefix = Gitlab::Database.postgresql? ? 'E' : ''
-    non_empty = "
-      (st_commits IS NOT NULL AND st_commits != #{literal_prefix}'--- []\n')
-      OR
-      (st_diffs IS NOT NULL AND st_diffs != #{literal_prefix}'--- []\n')
-    ".squish
-
-    MergeRequestDiff.where(non_empty).each_batch(of: 500) do |relation, index|
-      range = relation.pluck('MIN(id)', 'MAX(id)').first
-
-      Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits.new.perform(*range)
-    end
-  end
-
-  def down
-  end
-end

db/migrate/20180502122856_create_project_mirror_data.rb

@@ -4,17 +4,25 @@ class CreateProjectMirrorData < ActiveRecord::Migration[4.2]
   DOWNTIME = false
   def up
-    return if table_exists?(:project_mirror_data)
-
-    create_table :project_mirror_data do |t|
-      t.references :project, index: true, foreign_key: { on_delete: :cascade }
-      t.string :status
-      t.string :jid
-      t.text :last_error
+    if table_exists?(:project_mirror_data)
+      add_column :project_mirror_data, :status, :string unless column_exists?(:project_mirror_data, :status)
+      add_column :project_mirror_data, :jid, :string unless column_exists?(:project_mirror_data, :jid)
+      add_column :project_mirror_data, :last_error, :text unless column_exists?(:project_mirror_data, :last_error)
+    else
+      create_table :project_mirror_data do |t|
+        t.references :project, index: true, foreign_key: { on_delete: :cascade }
+        t.string :status
+        t.string :jid
+        t.text :last_error
+      end
     end
   end
   def down
-    drop_table(:project_mirror_data) if table_exists?(:project_mirror_data)
+    remove_column :project_mirror_data, :status
+    remove_column :project_mirror_data, :jid
+    remove_column :project_mirror_data, :last_error
+
+    # ee/db/migrate/20170509153720_create_project_mirror_data_ee.rb will remove the table.
   end
 end

db/migrate/20180503131624_create_remote_mirrors.rb

@@ -29,6 +29,6 @@ class CreateRemoteMirrors < ActiveRecord::Migration[4.2]
   end
   def down
-    drop_table(:remote_mirrors) if table_exists?(:remote_mirrors)
+    # ee/db/migrate/20160321161032_create_remote_mirrors_ee.rb will remove the table
   end
 end

db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb

@@ -10,6 +10,6 @@ class AddRemoteMirrorAvailableOverriddenToProjects < ActiveRecord::Migration[4.2
   end
   def down
-    remove_column(:projects, :remote_mirror_available_overridden) if column_exists?(:projects, :remote_mirror_available_overridden)
+    # ee/db/migrate/20171017130239_add_remote_mirror_available_overridden_to_projects_ee.rb will remove the column.
   end
 end

db/migrate/20180503193542_add_indexes_to_remote_mirror.rb

@@ -10,6 +10,7 @@ class AddIndexesToRemoteMirror < ActiveRecord::Migration[4.2]
   end
   def down
+    # ee/db/migrate/20170208144550_add_index_to_mirrors_last_update_at_fields.rb will remove the index.
     # rubocop:disable Migration/RemoveIndex
     remove_index :remote_mirrors, :last_successful_update_at if index_exists? :remote_mirrors, :last_successful_update_at
   end

db/migrate/20180503193953_add_mirror_available_to_application_settings.rb

@@ -10,6 +10,6 @@ class AddMirrorAvailableToApplicationSettings < ActiveRecord::Migration[4.2]
   end
   def down
-    remove_column(:application_settings, :mirror_available) if column_exists?(:application_settings, :mirror_available)
+    # ee/db/migrate/20171017125928_add_remote_mirror_available_to_application_settings.rb will remove the column.
   end
 end

db/migrate/20171216111734_clean_up_for_members.rb → db/migrate/20190403161806_update_designs_index.rb

+# frozen_string_literal: true
+
 # See http://doc.gitlab.com/ce/development/migration_style_guide.html
 # for more information on how to write migrations for GitLab.
-class CleanUpForMembers < ActiveRecord::Migration[4.2]
+class UpdateDesignsIndex < ActiveRecord::Migration[5.0]
   include Gitlab::Database::MigrationHelpers
   # Set this constant to true if this migration requires downtime.
@@ -9,23 +11,11 @@ class CleanUpForMembers < ActiveRecord::Migration[4.2]
   disable_ddl_transaction!
-  class Member < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'members'
-  end
-
   def up
-    condition = <<~EOF.squish
-      invite_token IS NULL AND
-      NOT EXISTS (SELECT 1 FROM users WHERE users.id = members.user_id)
-    EOF
-
-    Member.each_batch(of: 10_000) do |batch|
-      batch.where(condition).delete_all
-    end
+    remove_concurrent_index :design_management_designs, :issue_id, unique: true
   end
   def down
+    add_concurrent_index :design_management_designs, :issue_id, unique: true
   end
 end

db/migrate/20190409224933_add_name_to_geo_nodes.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddNameToGeoNodes < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def up
+    add_column :geo_nodes, :name, :string
+
+    # url is also unique, and its type and size is identical to the name column,
+    # so this is safe.
+    execute "UPDATE geo_nodes SET name = url;"
+
+    # url is also `null: false`, so this is safe.
+    change_column :geo_nodes, :name, :string, null: false
+  end
+
+  def down
+    remove_column :geo_nodes, :name
+  end
+end

db/post_migrate/20171005130944_schedule_create_gpg_key_subkeys_from_gpg_keys.rb → db/migrate/20190410173409_add_name_index_to_geo_nodes.rb

+# frozen_string_literal: true
+
 # See http://doc.gitlab.com/ce/development/migration_style_guide.html
 # for more information on how to write migrations for GitLab.
-class ScheduleCreateGpgKeySubkeysFromGpgKeys < ActiveRecord::Migration[4.2]
-  disable_ddl_transaction!
+class AddNameIndexToGeoNodes < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+  # Set this constant to true if this migration requires downtime.
   DOWNTIME = false
-  MIGRATION = 'CreateGpgKeySubkeysFromGpgKeys'
-
-  class GpgKey < ActiveRecord::Base
-    self.table_name = 'gpg_keys'
-    include EachBatch
-  end
+  disable_ddl_transaction!
   def up
-    GpgKey.select(:id).each_batch do |gpg_keys|
-      jobs = gpg_keys.pluck(:id).map do |id|
-        [MIGRATION, [id]]
-      end
-
-      BackgroundMigrationWorker.bulk_perform_async(jobs)
-    end
+    add_concurrent_index :geo_nodes, :name, unique: true
   end
   def down
+    remove_concurrent_index :geo_nodes, :name
   end
 end

db/migrate/20190412183653_remove_url_index_from_geo_nodes.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveUrlIndexFromGeoNodes < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index :geo_nodes, :url
+  end
+
+  def down
+    add_concurrent_index :geo_nodes, :url, unique: true
+  end
+end

db/migrate/20190414185432_add_comment_to_vulnerability_feedback.rb

+# frozen_string_literal: true
+
+class AddCommentToVulnerabilityFeedback < ActiveRecord::Migration[5.1]
+  DOWNTIME = false
+
+  def up
+    add_column :vulnerability_feedback, :comment_author_id, :integer
+    add_column :vulnerability_feedback, :comment, :text
+    add_column :vulnerability_feedback, :comment_timestamp, :datetime_with_timezone
+  end
+
+  def down
+    remove_column :vulnerability_feedback, :comment_author_id
+    remove_column :vulnerability_feedback, :comment
+    remove_column :vulnerability_feedback, :comment_timestamp
+  end
+end

db/migrate/20190415172035_update_insights_foreign_keys.rb

+# frozen_string_literal: true
+
+# rubocop: disable Migration/AddConcurrentForeignKey
+
+class UpdateInsightsForeignKeys < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    remove_foreign_key_if_exists(:insights, column: :project_id)
+    add_foreign_key(:insights, :projects, column: :project_id, on_delete: :cascade)
+
+    remove_foreign_key_if_exists(:insights, column: :namespace_id)
+    add_foreign_key(:insights, :namespaces, column: :namespace_id, on_delete: :cascade)
+  end
+
+  def down
+    remove_foreign_key_if_exists(:insights, column: :namespace_id)
+    add_foreign_key(:insights, :namespaces, column: :namespace_id)
+
+    remove_foreign_key_if_exists(:insights, column: :project_id)
+    add_foreign_key(:insights, :projects, column: :project_id)
+  end
+end

db/migrate/20190418132750_add_foreign_key_from_vulnerability_feedback_to_users.rb

+# frozen_string_literal: true
+
+class AddForeignKeyFromVulnerabilityFeedbackToUsers < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :vulnerability_feedback, :users, column: :comment_author_id, on_delete: :nullify
+    add_concurrent_index :vulnerability_feedback, :comment_author_id
+  end
+
+  def down
+    remove_foreign_key :vulnerability_feedback, column: :comment_author_id
+    remove_concurrent_index :vulnerability_feedback, :comment_author_id
+  end
+end

db/migrate/20190419121952_add_bridged_pipeline_id_to_bridges.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddBridgedPipelineIdToBridges < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :ci_builds, :upstream_pipeline_id, :integer
+  end
+end