Make Ci::ArchiveTraceWorker use replica for reads

This sets the data_consistency for the Ci::ArchiveTraceWorker to
sticky. Allowing the read queries before any write to be handled by a
replica.

The job is scheduled from the Ci::BuildFinishedWorker with some delay
built in. Which means this job can take the 0.8s wait in the
beginning, and is very likely to find a replica that is up to date
with the WAL locations at the time of scheduling.

To increase the effect of this, this also preloads some associations
in the worker. That way, we can take advantage of loading them from a
replica rather than doing it lazily from the primary.

app/models/ci/build.rb

@@ -144,6 +144,7 @@ def persisted_environment=(environment)
     scope :eager_load_job_artifacts, -> { includes(:job_artifacts) }
     scope :eager_load_tags, -> { includes(:tags) }
+    scope :eager_load_for_archiving_trace, -> { includes(:project, :pending_state) }
     scope :eager_load_everything, -> do
       includes(

app/workers/ci/archive_trace_worker.rb

@@ -4,13 +4,19 @@ module Ci
   class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker
     include ApplicationWorker
-    data_consistency :always
+    data_consistency :sticky, feature_flag: :sticky_ci_archive_trace_worker
     sidekiq_options retry: 3
     include PipelineBackgroundQueue
     def perform(job_id)
-      Ci::Build.without_archived_trace.find_by_id(job_id).try do |job|
+      archivable_jobs = Ci::Build.without_archived_trace
+
+      if Feature.enabled?(:sticky_ci_archive_trace_worker)
+        archivable_jobs = archivable_jobs.eager_load_for_archiving_trace
+      end
+
+      archivable_jobs.find_by_id(job_id).try do |job|
         Ci::ArchiveTraceService.new.execute(job, worker_name: self.class.name)
       end
     end

config/feature_flags/development/sticky_ci_archive_trace_worker.yml

+---
+name: sticky_ci_archive_trace_worker
+introduced_by_url:
+rollout_issue_url: https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1681
+milestone: '15.0'
+type: development
+group: group::scalability
+default_enabled: false

spec/workers/ci/archive_trace_worker_spec.rb

@@ -16,6 +16,34 @@
         subject
       end
+
+      it 'has preloaded the arguments for archiving' do
+        allow_next_instance_of(Ci::ArchiveTraceService) do |instance|
+          allow(instance).to receive(:execute) do |job|
+            expect(job.association(:project)).to be_loaded
+            expect(job.association(:pending_state)).to be_loaded
+          end
+        end
+
+        subject
+      end
+
+      context 'when sticky_ci_archive_trace_worker is disabled' do
+        before do
+          stub_feature_flags(sticky_ci_archive_trace_worker: false)
+        end
+
+        it 'does not preload associations' do
+          allow_next_instance_of(Ci::ArchiveTraceService) do |instance|
+            allow(instance).to receive(:execute) do |job|
+              expect(job.association(:project)).not_to be_loaded
+              expect(job.association(:pending_state)).not_to be_loaded
+            end
+          end
+
+          subject
+        end
+      end
     end
     context 'when job is not found' do