Merge branch '498854-delete-orphan-variables' into 'master'

Delete orphaned pipeline variables

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169441



Merged-by: Tiger Watson <twatson@gitlab.com>
Approved-by: Max Orefice <morefice@gitlab.com>
Approved-by: Tiger Watson <twatson@gitlab.com>
Reviewed-by: Max Orefice <morefice@gitlab.com>
Co-authored-by: Marius Bobin <mbobin@gitlab.com>

db/docs/batched_background_migrations/delete_orphaned_pipeline_variable_records.yml

+---
+migration_job_name: DeleteOrphanedPipelineVariableRecords
+description: Deletes corrupted rows from p_ci_pipeline_variables table
+feature_category: continuous_integration
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169441
+milestone: '17.6'
+queued_migration_version: 20241016131601
+finalized_by: # version of the migration that finalized this BBM

db/post_migrate/20241016131601_queue_delete_orphaned_pipeline_variable_records.rb

+# frozen_string_literal: true
+
+class QueueDeleteOrphanedPipelineVariableRecords < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+  restrict_gitlab_migration gitlab_schema: :gitlab_ci
+
+  MIGRATION = "DeleteOrphanedPipelineVariableRecords"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :p_ci_pipeline_variables,
+      :pipeline_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      batch_class_name: 'LooseIndexScanBatchingStrategy',
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :p_ci_pipeline_variables, :pipeline_id, [])
+  end
+end

db/schema_migrations/20241016131601

+41127c2fcc61d93b91ac88a9f21c40ed503b18a25140c7ef6a30ec37d83f1f54
\ No newline at end of file

lib/gitlab/background_migration/delete_orphaned_pipeline_variable_records.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class DeleteOrphanedPipelineVariableRecords < BatchedMigrationJob
+      operation_name :delete_orphaned_pipeline_variable_records
+      feature_category :continuous_integration
+
+      class CiPipeline < ::Ci::ApplicationRecord
+        self.table_name = :p_ci_pipelines
+        self.primary_key = :id
+      end
+
+      def perform
+        distinct_each_batch do |batch|
+          pipeline_ids = batch.pluck(batch_column)
+          pipelines_query = CiPipeline
+            .where('p_ci_pipeline_variables.pipeline_id = p_ci_pipelines.id')
+            .where('p_ci_pipeline_variables.partition_id = p_ci_pipelines.partition_id')
+            .select(1)
+
+          base_relation
+            .where(batch_column => pipeline_ids)
+            .where('NOT EXISTS (?)', pipelines_query)
+            .delete_all
+        end
+      end
+
+      private
+
+      def base_relation
+        define_batchable_model(batch_table, connection: connection, primary_key: :id)
+          .where(batch_column => start_id..end_id)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/delete_orphaned_pipeline_variable_records_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedPipelineVariableRecords,
+  feature_category: :continuous_integration, migration: :gitlab_ci do
+  let(:pipelines_table) { table(:p_ci_pipelines, database: :ci, primary_key: :id) }
+  let(:variables_table) { table(:p_ci_pipeline_variables, database: :ci, primary_key: :id) }
+
+  let(:default_attributes) { { project_id: 600, partition_id: 100 } }
+  let!(:regular_pipeline) { pipelines_table.create!(default_attributes) }
+  let!(:deleted_pipeline) { pipelines_table.create!(default_attributes) }
+  let!(:other_pipeline) { pipelines_table.create!(default_attributes) }
+
+  let!(:regular_variable) do
+    variables_table.create!(pipeline_id: regular_pipeline.id, key: :key1, **default_attributes)
+  end
+
+  let!(:orphaned_variable) do
+    variables_table.create!(pipeline_id: deleted_pipeline.id, key: :key2, **default_attributes)
+  end
+
+  let(:connection) { Ci::ApplicationRecord.connection }
+
+  around do |example|
+    connection.transaction do
+      connection.execute(<<~SQL)
+        ALTER TABLE ci_pipelines DISABLE TRIGGER ALL;
+      SQL
+
+      example.run
+
+      connection.execute(<<~SQL)
+        ALTER TABLE ci_pipelines ENABLE TRIGGER ALL;
+      SQL
+    end
+  end
+
+  describe '#perform' do
+    subject(:migration) do
+      described_class.new(
+        start_id: variables_table.minimum(:pipeline_id),
+        end_id: variables_table.maximum(:pipeline_id),
+        batch_table: :p_ci_pipeline_variables,
+        batch_column: :pipeline_id,
+        sub_batch_size: 100,
+        pause_ms: 0,
+        connection: connection
+      )
+    end
+
+    it 'deletes from p_ci_pipeline_variables where pipeline_id has no related', :aggregate_failures do
+      expect { deleted_pipeline.delete }.to not_change { variables_table.count }
+
+      expect { migration.perform }.to change { variables_table.count }.from(2).to(1)
+
+      expect(regular_variable.reload).to be_persisted
+      expect { orphaned_variable.reload }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+  end
+end

spec/migrations/20241016131601_queue_delete_orphaned_pipeline_variable_records_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueDeleteOrphanedPipelineVariableRecords, migration: :gitlab_ci, feature_category: :continuous_integration do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :p_ci_pipeline_variables,
+          column_name: :pipeline_id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE,
+          gitlab_schema: :gitlab_ci
+        )
+      }
+    end
+  end
+end