Merge branch 'catalog-resources-sync-worker-cron' into 'master'

Add cron to run catalog resources ProcessSyncEventsWorker every minute

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138865



Merged-by: Laura Montemayor <lmontemayor@gitlab.com>
Approved-by: Max Orefice <morefice@gitlab.com>
Approved-by: Laura Montemayor <lmontemayor@gitlab.com>
Reviewed-by: Max Orefice <morefice@gitlab.com>
Co-authored-by: lma-git <lma@gitlab.com>

app/workers/all_queues.yml

@@ -246,6 +246,15 @@
   :weight: 1
   :idempotent: false
   :tags: []
+- :name: cronjob:ci_catalog_resources_process_sync_events
+  :worker_name: Ci::Catalog::Resources::ProcessSyncEventsWorker
+  :feature_category: :pipeline_composition
+  :has_external_dependencies: false
+  :urgency: :high
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:ci_delete_unit_tests
   :worker_name: Ci::DeleteUnitTestsWorker
   :feature_category: :code_testing
@@ -2694,15 +2703,6 @@
   :weight: 1
   :idempotent: true
   :tags: []
-- :name: ci_catalog_resources_process_sync_events
-  :worker_name: Ci::Catalog::Resources::ProcessSyncEventsWorker
-  :feature_category: :pipeline_composition
-  :has_external_dependencies: false
-  :urgency: :high
-  :resource_boundary: :unknown
-  :weight: 1
-  :idempotent: true
-  :tags: []
 - :name: ci_delete_objects
   :worker_name: Ci::DeleteObjectsWorker
   :feature_category: :continuous_integration

app/workers/ci/catalog/resources/process_sync_events_worker.rb

@@ -5,8 +5,18 @@ module Catalog
     module Resources
       # This worker can be called multiple times simultaneously but only one can process events
       # at a time. This is ensured by `try_obtain_lease` in `Ci::ProcessSyncEventsService`.
+      #
+      # This worker is enqueued in 3 ways:
+      # 1. By Project model callback after updating one of the columns referenced in
+      #    `Ci::Catalog::Resource#sync_with_project`.
+      # 2. Every minute by cron job. This ensures we process SyncEvents from direct/bulk
+      #    database updates that do not use the Project AR model.
+      # 3. By `Ci::ProcessSyncEventsService` if there are any remaining pending
+      #    SyncEvents after processing.
+      #
       class ProcessSyncEventsWorker
         include ApplicationWorker
+        include CronjobQueue # rubocop: disable Scalability/CronWorkerContext -- Periodic processing is required
         feature_category :pipeline_composition
@@ -17,6 +27,8 @@ class ProcessSyncEventsWorker
         deduplicate :until_executed, if_deduplicated: :reschedule_once, ttl: 1.minute
         def perform
+          return if Feature.disabled?(:ci_process_catalog_resource_sync_events)
+
           results = ::Ci::ProcessSyncEventsService.new(
             ::Ci::Catalog::Resources::SyncEvent, ::Ci::Catalog::Resource
           ).execute

config/initializers/1_settings.rb

@@ -700,6 +700,9 @@
 Settings.cron_jobs['ci_schedule_unlock_pipelines_in_queue_worker'] ||= {}
 Settings.cron_jobs['ci_schedule_unlock_pipelines_in_queue_worker']['cron'] ||= '*/1 * * * *'
 Settings.cron_jobs['ci_schedule_unlock_pipelines_in_queue_worker']['job_class'] = 'Ci::ScheduleUnlockPipelinesInQueueCronWorker'
+Settings.cron_jobs['ci_catalog_resources_process_sync_events_worker'] ||= {}
+Settings.cron_jobs['ci_catalog_resources_process_sync_events_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['ci_catalog_resources_process_sync_events_worker']['job_class'] = 'Ci::Catalog::Resources::ProcessSyncEventsWorker'
 Gitlab.ee do
   Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= {}

config/sidekiq_queues.yml

@@ -157,8 +157,6 @@
   - 1
 - - ci_cancel_redundant_pipelines
   - 1
-- - ci_catalog_resources_process_sync_events
-  - 1
 - - ci_delete_objects
   - 1
 - - ci_initialize_pipelines_iid_sequence

spec/workers/ci/catalog/resources/process_sync_events_worker_spec.rb

@@ -16,14 +16,14 @@
   end
   describe '#perform' do
-    let_it_be(:project) { create(:project) }
+    let_it_be(:project) { create(:project, name: 'Old Name') }
     let_it_be(:resource) { create(:ci_catalog_resource, project: project) }
     before_all do
       create(:ci_catalog_resource_sync_event, catalog_resource: resource, status: :processed)
       create_list(:ci_catalog_resource_sync_event, 2, catalog_resource: resource)
       # PG trigger adds an event for this update
-      project.update!(name: 'Name', description: 'Test', visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+      project.update!(name: 'New Name', description: 'Test', visibility_level: Gitlab::VisibilityLevel::INTERNAL)
     end
     subject(:perform) { worker.perform }
@@ -48,5 +48,21 @@
       perform
     end
+
+    context 'when FF `ci_process_catalog_resource_sync_events` is disabled' do
+      before do
+        stub_feature_flags(ci_process_catalog_resource_sync_events: false)
+      end
+
+      it 'does not process the sync events', :aggregate_failures do
+        expect(worker).not_to receive(:log_extra_metadata_on_done)
+
+        expect { perform }.not_to change { Ci::Catalog::Resources::SyncEvent.status_pending.count }
+
+        expect(resource.reload.name).to eq('Old Name')
+        expect(resource.reload.description).to be_nil
+        expect(resource.reload.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE)
+      end
+    end
   end
 end