Reduce pipeline persistent ref creation

This commit reduces the persistent ref creation
for pipelines.

app/models/ci/build.rb

@@ -74,6 +74,7 @@ class Build < Ci::Processable
     delegate :gitlab_deploy_token, to: :project
     delegate :harbor_integration, to: :project
     delegate :trigger_short_token, to: :trigger_request, allow_nil: true
+    delegate :ensure_persistent_ref, to: :pipeline
     ##
     # Since Gitlab 11.5, deployments records started being created right after
@@ -325,7 +326,11 @@ def clone_accessors
       after_transition pending: :running do |build|
         build.run_after_commit do
-          build.pipeline.persistent_ref.create
+          if ::Feature.enabled?(:ci_reduce_persistent_ref_writes, build.project, default_enabled: :yaml)
+            build.ensure_persistent_ref
+          else
+            build.pipeline.persistent_ref.create
+          end
           BuildHooksWorker.perform_async(id)
         end

app/models/ci/pipeline.rb

@@ -253,7 +253,9 @@ class Pipeline < Ci::ApplicationRecord
       after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline|
         pipeline.run_after_commit do
-          pipeline.persistent_ref.delete
+          if ::Feature.disabled?(:ci_reduce_persistent_ref_writes, pipeline.project, default_enabled: :yaml)
+            pipeline.persistent_ref.delete
+          end
           pipeline.all_merge_requests.each do |merge_request|
             next unless merge_request.auto_merge_enabled?
@@ -288,6 +290,14 @@ class Pipeline < Ci::ApplicationRecord
         end
       end
+      after_transition any => ::Ci::Pipeline.stopped_statuses do |pipeline|
+        pipeline.run_after_commit do
+          if ::Feature.enabled?(:ci_reduce_persistent_ref_writes, pipeline.project, default_enabled: :yaml)
+            pipeline.persistent_ref.delete
+          end
+        end
+      end
+
       after_transition any => [:success, :failed] do |pipeline|
         ref_status = pipeline.ci_ref&.update_status_by!(pipeline)
@@ -1257,6 +1267,12 @@ def ensure_ci_ref!
       self.ci_ref = Ci::Ref.ensure_for(self)
     end
+    def ensure_persistent_ref
+      return if persistent_ref.exist?
+
+      persistent_ref.create
+    end
+
     def reset_source_bridge!(current_user)
       return unless bridge_waiting?

app/models/concerns/ci/has_status.rb

@@ -13,6 +13,7 @@ module HasStatus
     STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze
     ACTIVE_STATUSES = %w[waiting_for_resource preparing pending running].freeze
     COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
+    STOPPED_STATUSES = COMPLETED_STATUSES + BLOCKED_STATUS
     ORDERED_STATUSES = %w[failed preparing pending running waiting_for_resource manual scheduled canceled success skipped created].freeze
     PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze
     EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze
@@ -47,6 +48,10 @@ def all_state_names
       def completed_statuses
         COMPLETED_STATUSES.map(&:to_sym)
       end
+
+      def stopped_statuses
+        STOPPED_STATUSES.map(&:to_sym)
+      end
     end
     included do

app/services/ci/pipeline_creation/start_pipeline_service.rb

@@ -10,6 +10,13 @@ def initialize(pipeline)
       end
       def execute
+        ##
+        # Create a persistent ref for the pipeline.
+        # The pipeline ref is fetched in the jobs and deleted when the pipeline transitions to a finished state.
+        if ::Feature.enabled?(:ci_reduce_persistent_ref_writes, pipeline.project, default_enabled: :yaml)
+          pipeline.ensure_persistent_ref
+        end
+
         Ci::ProcessPipelineService.new(pipeline).execute
       end
     end

config/feature_flags/development/ci_reduce_persistent_ref_writes.yml

+---
+name: ci_reduce_persistent_ref_writes
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83730
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/354837
+milestone: '14.10'
+type: development
+group: group::pipeline execution
+default_enabled: false

spec/models/ci/build_spec.rb

@@ -3733,6 +3733,7 @@ def run_job_without_exception
     context 'for pipeline ref existence' do
       it 'ensures pipeline ref creation' do
+        expect(job.pipeline).to receive(:ensure_persistent_ref).once.and_call_original
         expect(job.pipeline.persistent_ref).to receive(:create).once
         run_job_without_exception
@@ -3745,6 +3746,19 @@ def run_job_without_exception
         run_job_without_exception
       end
+
+      context 'when ci_reduce_persistent_ref_writes feature flag is disabled' do
+        before do
+          stub_feature_flags(ci_reduce_persistent_ref_writes: false)
+        end
+
+        it 'falls back to the previous behavior' do
+          expect(job.pipeline).not_to receive(:ensure_persistent_ref)
+          expect(job.pipeline.persistent_ref).to receive(:create).once
+
+          run_job_without_exception
+        end
+      end
     end
     shared_examples 'saves data on transition' do

spec/models/ci/pipeline_spec.rb

@@ -1423,7 +1423,7 @@ def create_build(name, status)
     let(:build_b) { create_build('build2', queued_at: 0) }
     let(:build_c) { create_build('build3', queued_at: 0) }
-    %w[succeed! drop! cancel! skip!].each do |action|
+    %w[succeed! drop! cancel! skip! block! delay!].each do |action|
       context "when the pipeline recieved #{action} event" do
         it 'deletes a persistent ref' do
           expect(pipeline.persistent_ref).to receive(:delete).once
@@ -1433,6 +1433,32 @@ def create_build(name, status)
       end
     end
+    context 'when ci_reduce_persistent_ref_writes feature flag is disabled' do
+      before do
+        stub_feature_flags(ci_reduce_persistent_ref_writes: false)
+      end
+
+      %w[succeed! drop! cancel! skip!].each do |action|
+        context "when the pipeline recieved #{action} event" do
+          it 'deletes a persistent ref' do
+            expect(pipeline.persistent_ref).to receive(:delete).once
+
+            pipeline.public_send(action)
+          end
+        end
+      end
+
+      %w[block! delay!].each do |action|
+        context "when the pipeline recieved #{action} event" do
+          it 'does not delete a persistent ref' do
+            expect(pipeline.persistent_ref).not_to receive(:delete)
+
+            pipeline.public_send(action)
+          end
+        end
+      end
+    end
+
     describe 'synching status to Jira' do
       let(:worker) { ::JiraConnect::SyncBuildsWorker }
@@ -1828,6 +1854,32 @@ def create_build(name, *traits, queued_at: current, started_from: 0, **opts)
     end
   end
+  describe '#ensure_persistent_ref' do
+    subject { pipeline.ensure_persistent_ref }
+
+    let(:pipeline) { create(:ci_pipeline, project: project) }
+
+    context 'when the persistent ref does not exist' do
+      it 'creates a ref' do
+        expect(pipeline.persistent_ref).to receive(:create).once
+
+        subject
+      end
+    end
+
+    context 'when the persistent ref exists' do
+      before do
+        pipeline.persistent_ref.create # rubocop:disable Rails/SaveBang
+      end
+
+      it 'does not create a ref' do
+        expect(pipeline.persistent_ref).not_to receive(:create)
+
+        subject
+      end
+    end
+  end
+
   describe '#branch?' do
     subject { pipeline.branch? }

spec/services/ci/pipeline_creation/start_pipeline_service_spec.rb

@@ -16,5 +16,23 @@
       service.execute
     end
+
+    it 'creates pipeline ref' do
+      expect(pipeline.persistent_ref).to receive(:create).once
+
+      service.execute
+    end
+
+    context 'when ci_reduce_persistent_ref_writes feature flag is disabled' do
+      before do
+        stub_feature_flags(ci_reduce_persistent_ref_writes: false)
+      end
+
+      it 'does not populate pipeline ref' do
+        expect(pipeline.persistent_ref).not_to receive(:create)
+
+        service.execute
+      end
+    end
   end
 end