Merge branch '3269-speed-up-cross-db-pluck' into 'master'

Improve Geo repository sync performance for larger databases

Closes #3269

See merge request !2887

app/models/geo_node.rb

@@ -138,6 +138,27 @@ def project_registries
     end
   end
+  # These are projects that meet the project restriction but haven't yet been
+  # synced (i.e., do not yet have a project registry entry).
+  #
+  # This query requires data from two different databases, and unavoidably
+  # plucks a list of project IDs from one into the other. This will not scale
+  # well with the number of synchronized projects - the query will increase
+  # linearly in size - so this should be replaced with postgres_fdw ASAP.
+  def unsynced_projects
+    registry_project_ids = project_registries.pluck(:project_id)
+    return projects if registry_project_ids.empty?
+
+    joined_relation = projects.joins(<<~SQL)
+      LEFT OUTER JOIN
+      (VALUES #{registry_project_ids.map { |id| "(#{id}, 't')" }.join(',')})
+      project_registry(project_id, registry_present)
+      ON projects.id = project_registry.project_id
+    SQL
+
+    joined_relation.where(project_registry: { registry_present: [nil, false] })
+  end
+
   def uploads
     if restricted_project_ids
       uploads_table   = Upload.arel_table

app/workers/geo/repository_sync_worker.rb

@@ -23,11 +23,10 @@ def load_pending_resources
     end
     def find_project_ids_not_synced
-      current_node.projects
-                  .where.not(id: Geo::ProjectRegistry.pluck(:project_id))
-                  .order(last_repository_updated_at: :desc)
-                  .limit(db_retrieve_batch_size)
-                  .pluck(:id)
+      current_node.unsynced_projects
+        .reorder(last_repository_updated_at: :desc)
+        .limit(db_retrieve_batch_size)
+        .pluck(:id)
     end
     def find_project_ids_updated_recently

changelogs/unreleased-ee/3269-speed-up-cross-db-pluck.yml

+---
+title: Improve Geo repository sync performance for larger databases
+merge_request: 2887
+author:
+type: fixed

spec/workers/geo/repository_sync_worker_spec.rb

 require 'spec_helper'
-describe Geo::RepositorySyncWorker do
+describe Geo::RepositorySyncWorker, :postgresql do
   let!(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
   let!(:secondary) { create(:geo_node, :current) }
   let(:synced_group) { create(:group) }