Merge branch...

Merge branch 'backfill-desired-sharding-key-small-table-dependency_proxy_blob_states' into 'master' 

Add and backfill group_id for dependency_proxy_blob_states

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169237



Merged-by: Omar Qunsul <oqunsul@gitlab.com>
Approved-by: Omar Qunsul <oqunsul@gitlab.com>
Co-authored-by: Shubham Kumar <shukumar@gitlab.com>

db/docs/batched_background_migrations/backfill_dependency_proxy_blob_states_group_id.yml

+---
+migration_job_name: BackfillDependencyProxyBlobStatesGroupId
+description: Backfills sharding key `dependency_proxy_blob_states.group_id` from `dependency_proxy_blobs`.
+feature_category: geo_replication
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169237
+milestone: '17.6'
+queued_migration_version: 20241015075957
+finalized_by: # version of the migration that finalized this BBM

db/docs/dependency_proxy_blob_states.yml

@@ -19,3 +19,4 @@ desired_sharding_key:
         table: dependency_proxy_blobs
         sharding_key: group_id
         belongs_to: dependency_proxy_blob
+desired_sharding_key_migration_job_name: BackfillDependencyProxyBlobStatesGroupId

db/migrate/20241015075953_add_group_id_to_dependency_proxy_blob_states.rb

+# frozen_string_literal: true
+
+class AddGroupIdToDependencyProxyBlobStates < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+
+  def change
+    add_column :dependency_proxy_blob_states, :group_id, :bigint
+  end
+end

db/post_migrate/20241015075954_index_dependency_proxy_blob_states_on_group_id.rb

+# frozen_string_literal: true
+
+class IndexDependencyProxyBlobStatesOnGroupId < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_dependency_proxy_blob_states_on_group_id'
+
+  def up
+    add_concurrent_index :dependency_proxy_blob_states, :group_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :dependency_proxy_blob_states, INDEX_NAME
+  end
+end

db/post_migrate/20241015075955_add_dependency_proxy_blob_states_group_id_fk.rb

+# frozen_string_literal: true
+
+class AddDependencyProxyBlobStatesGroupIdFk < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :dependency_proxy_blob_states, :namespaces, column: :group_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :dependency_proxy_blob_states, column: :group_id
+    end
+  end
+end

db/post_migrate/20241015075956_add_dependency_proxy_blob_states_group_id_trigger.rb

+# frozen_string_literal: true
+
+class AddDependencyProxyBlobStatesGroupIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :dependency_proxy_blob_states,
+      sharding_key: :group_id,
+      parent_table: :dependency_proxy_blobs,
+      parent_sharding_key: :group_id,
+      foreign_key: :dependency_proxy_blob_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :dependency_proxy_blob_states,
+      sharding_key: :group_id,
+      parent_table: :dependency_proxy_blobs,
+      parent_sharding_key: :group_id,
+      foreign_key: :dependency_proxy_blob_id
+    )
+  end
+end

db/post_migrate/20241015075957_queue_backfill_dependency_proxy_blob_states_group_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillDependencyProxyBlobStatesGroupId < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+  MIGRATION = "BackfillDependencyProxyBlobStatesGroupId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :dependency_proxy_blob_states,
+      :dependency_proxy_blob_id,
+      :group_id,
+      :dependency_proxy_blobs,
+      :group_id,
+      :dependency_proxy_blob_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :dependency_proxy_blob_states,
+      :dependency_proxy_blob_id,
+      [
+        :group_id,
+        :dependency_proxy_blobs,
+        :group_id,
+        :dependency_proxy_blob_id
+      ]
+    )
+  end
+end

db/schema_migrations/20241015075953

+bd0cac7a4cd41fad0dfdf2959f52ad3d2fb37563df9bb5ac3c5ab2c1c592f9d3
\ No newline at end of file

db/schema_migrations/20241015075954

+be10cd330779b21118c96f3e940e3632e8d79566eeb38af0c9118c0ab8507102
\ No newline at end of file

db/schema_migrations/20241015075955

+e384696ed03e4ddb5b88941cf0141cb9793c66eebcb6892e1cee3ed1225201aa
\ No newline at end of file

db/schema_migrations/20241015075956

+33ad1996ec23c69a5295ab18b7ba1b986e4d1f3e05de123178e5c864e531fb6b
\ No newline at end of file

db/schema_migrations/20241015075957

+bad4fab793a81707348b46027bee797eaa641ddb71e17d1c22950887dc590d2e
\ No newline at end of file

db/structure.sql

@@ -1275,6 +1275,22 @@ RETURN NEW;
 END
 $$;
+CREATE FUNCTION trigger_38bfee591e40() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+IF NEW."group_id" IS NULL THEN
+  SELECT "group_id"
+  INTO NEW."group_id"
+  FROM "dependency_proxy_blobs"
+  WHERE "dependency_proxy_blobs"."id" = NEW."dependency_proxy_blob_id";
+END IF;
+
+RETURN NEW;
+
+END
+$$;
+
 CREATE FUNCTION trigger_3d1a58344b29() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -10507,6 +10523,7 @@ CREATE TABLE dependency_proxy_blob_states (
     verification_retry_count smallint DEFAULT 0 NOT NULL,
     verification_checksum bytea,
     verification_failure text,
+    group_id bigint,
     CONSTRAINT check_8e4f76fffe CHECK ((char_length(verification_failure) <= 255))
 );
@@ -28870,6 +28887,8 @@ CREATE INDEX index_dependency_proxy_blob_states_needs_verification ON dependency
 CREATE INDEX index_dependency_proxy_blob_states_on_dependency_proxy_blob_id ON dependency_proxy_blob_states USING btree (dependency_proxy_blob_id);
+CREATE INDEX index_dependency_proxy_blob_states_on_group_id ON dependency_proxy_blob_states USING btree (group_id);
+
 CREATE INDEX index_dependency_proxy_blob_states_on_verification_state ON dependency_proxy_blob_states USING btree (verification_state);
 CREATE INDEX index_dependency_proxy_blob_states_pending_verification ON dependency_proxy_blob_states USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
@@ -33868,6 +33887,8 @@ CREATE TRIGGER trigger_3691f9f6a69f BEFORE INSERT OR UPDATE ON remote_developmen
 CREATE TRIGGER trigger_388de55cd36c BEFORE INSERT OR UPDATE ON ci_builds_runner_session FOR EACH ROW EXECUTE FUNCTION trigger_388de55cd36c();
+CREATE TRIGGER trigger_38bfee591e40 BEFORE INSERT OR UPDATE ON dependency_proxy_blob_states FOR EACH ROW EXECUTE FUNCTION trigger_38bfee591e40();
+
 CREATE TRIGGER trigger_3d1a58344b29 BEFORE INSERT OR UPDATE ON alert_management_alert_assignees FOR EACH ROW EXECUTE FUNCTION trigger_3d1a58344b29();
 CREATE TRIGGER trigger_3e067fa9bfe3 BEFORE INSERT OR UPDATE ON incident_management_timeline_event_tag_links FOR EACH ROW EXECUTE FUNCTION trigger_3e067fa9bfe3();
@@ -34998,6 +35019,9 @@ ALTER TABLE ONLY milestones
 ALTER TABLE ONLY boards_epic_list_user_preferences
     ADD CONSTRAINT fk_95eac55851 FOREIGN KEY (epic_list_id) REFERENCES boards_epic_lists(id) ON DELETE CASCADE;
+ALTER TABLE ONLY dependency_proxy_blob_states
+    ADD CONSTRAINT fk_95ee495fd6 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY issues
     ADD CONSTRAINT fk_96b1dd429c FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE SET NULL;

lib/gitlab/background_migration/backfill_dependency_proxy_blob_states_group_id.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class BackfillDependencyProxyBlobStatesGroupId < BackfillDesiredShardingKeyJob
+      operation_name :backfill_dependency_proxy_blob_states_group_id
+      feature_category :geo_replication
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_dependency_proxy_blob_states_group_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillDependencyProxyBlobStatesGroupId,
+  feature_category: :geo_replication,
+  schema: 20241015075953 do
+  include_examples 'desired sharding key backfill job' do
+    let(:batch_table) { :dependency_proxy_blob_states }
+    let(:backfill_column) { :group_id }
+    let(:batch_column) { :dependency_proxy_blob_id }
+    let(:backfill_via_table) { :dependency_proxy_blobs }
+    let(:backfill_via_column) { :group_id }
+    let(:backfill_via_foreign_key) { :dependency_proxy_blob_id }
+  end
+end

spec/migrations/20241015075957_queue_backfill_dependency_proxy_blob_states_group_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillDependencyProxyBlobStatesGroupId, feature_category: :geo_replication do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :dependency_proxy_blob_states,
+          column_name: :dependency_proxy_blob_id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE,
+          gitlab_schema: :gitlab_main_cell,
+          job_arguments: [
+            :group_id,
+            :dependency_proxy_blobs,
+            :group_id,
+            :dependency_proxy_blob_id
+          ]
+        )
+      }
+    end
+  end
+end