Support AS MATERIALIZED in PG12

This change adds Arel support for the new CTE query syntax and
updates raw queries to use MATERIALIZED (when supported).

app/services/issue_rebalancing_service.rb

@@ -62,7 +62,7 @@ def update_positions(pairs_with_position, query_name)
   def run_update_query(values, query_name)
     Issue.connection.exec_query(<<~SQL, query_name)
-      WITH cte(cte_id, new_pos) AS (
+      WITH cte(cte_id, new_pos) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
        SELECT *
        FROM (VALUES #{values}) as t (id, pos)
       )

changelogs/unreleased/245323-arel-support-for-materialized-cte.yml

+---
+title: Add support for the MATERIALIZED keyword when using WITH (CTE) queries in PostgreSQL 12
+merge_request: 56976
+author:
+type: other

config/initializers/postgres_cte_as_materialized.rb

+# frozen_string_literal: true
+
+# This patch adds support for AS MATERIALIZED in Arel, see Gitlab::Database::AsWithMaterialized for more info
+module Arel
+  module Visitors
+    class Arel::Visitors::PostgreSQL
+      def visit_Gitlab_Database_AsWithMaterialized(obj, collector) # rubocop:disable Naming/MethodName
+        collector = visit obj.left, collector
+        collector << " AS#{obj.expr} "
+        visit obj.right, collector
+      end
+    end
+  end
+end

config/initializers/postgresql_cte.rb

@@ -121,6 +121,8 @@ def build_with(arel)
           end
         when Arel::Nodes::As
           with_value
+        when Gitlab::Database::AsWithMaterialized
+          with_value
         end
       end

db/migrate/20200609212701_add_incident_settings_to_all_existing_projects.rb

@@ -8,7 +8,7 @@ def up
     # to preserve behavior for existing projects that
     # are using the create issue functionality with the default setting of true
     query = <<-SQL
-      WITH project_ids AS (
+      WITH project_ids AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}(
         SELECT DISTINCT issues.project_id AS id
         FROM issues
         LEFT OUTER JOIN project_incident_management_settings

db/post_migrate/20191105094625_set_report_type_for_vulnerabilities.rb

@@ -7,7 +7,7 @@ def up
     # set report_type based on vulnerability_occurrences from which the vulnerabilities were promoted,
     # that is, first vulnerability_occurrences among those having the same vulnerability_id
     execute <<~SQL
-    WITH first_findings_for_vulnerabilities AS (
+    WITH first_findings_for_vulnerabilities AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
       SELECT MIN(id) AS id, vulnerability_id
       FROM vulnerability_occurrences
       WHERE vulnerability_id IS NOT NULL

db/post_migrate/20191114173624_set_resolved_state_on_vulnerabilities.rb

@@ -6,7 +6,7 @@ class SetResolvedStateOnVulnerabilities < ActiveRecord::Migration[5.2]
   def up
     execute <<~SQL
     -- selecting IDs for all non-orphan Findings that either have no feedback or it's a non-dismissal feedback
-    WITH resolved_vulnerability_ids AS (
+    WITH resolved_vulnerability_ids AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
       SELECT DISTINCT vulnerability_id AS id
       FROM vulnerability_occurrences
       LEFT JOIN vulnerability_feedback ON vulnerability_feedback.project_fingerprint = ENCODE(vulnerability_occurrences.project_fingerprint::bytea, 'HEX')

db/post_migrate/20200305082754_remove_duplicate_labels_from_project.rb

@@ -55,7 +55,7 @@ def remove_full_duplicates(start_id, stop_id)
     # project_id title template description type color
     duplicate_labels = ApplicationRecord.connection.execute(<<-SQL.squish)
-WITH data AS (
+WITH data AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
   SELECT labels.*,
   row_number() OVER (PARTITION BY labels.project_id, labels.title, labels.template, labels.description, labels.type, labels.color ORDER BY labels.id) AS row_number,
   #{CREATE} AS restore_action
@@ -83,7 +83,7 @@ def rename_partial_duplicates(start_id, stop_id)
     # then add `_duplicate#{ID}`
     soft_duplicates = ApplicationRecord.connection.execute(<<-SQL.squish)
-WITH data AS (
+WITH data AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
   SELECT
      *,
      substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text as new_title,
@@ -108,7 +108,7 @@ def rename_partial_duplicates(start_id, stop_id)
   def restore_renamed_labels(start_id, stop_id)
     # the backup label IDs are not incremental, they are copied directly from the Labels table
     ApplicationRecord.connection.execute(<<-SQL.squish)
-WITH backups AS (
+WITH backups AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
   SELECT id, title
   FROM backup_labels
   WHERE project_id BETWEEN #{start_id} AND #{stop_id} AND

db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb

@@ -59,7 +59,7 @@ def remove_full_duplicates(start_id, stop_id)
     # group_id title template description type color
     duplicate_labels = ApplicationRecord.connection.execute(<<-SQL.squish)
-WITH data AS (
+WITH data AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
   SELECT labels.*,
   row_number() OVER (PARTITION BY labels.group_id, labels.title, labels.template, labels.description, labels.type, labels.color ORDER BY labels.id) AS row_number,
   #{CREATE} AS restore_action
@@ -87,7 +87,7 @@ def rename_partial_duplicates(start_id, stop_id)
     # then add `_duplicate#{ID}`
     soft_duplicates = ApplicationRecord.connection.execute(<<-SQL.squish)
-WITH data AS (
+WITH data AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
   SELECT
      *,
      substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text as new_title,
@@ -112,7 +112,7 @@ def rename_partial_duplicates(start_id, stop_id)
   def restore_renamed_labels(start_id, stop_id)
     # the backup label IDs are not incremental, they are copied directly from the Labels table
     ApplicationRecord.connection.execute(<<-SQL.squish)
-WITH backups AS (
+WITH backups AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
   SELECT id, title
   FROM backup_labels
   WHERE id BETWEEN #{start_id} AND #{stop_id}

db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb

@@ -26,7 +26,7 @@ def up
       min, max = relation.pluck('MIN(job_id)', 'MAX(job_id)').flatten
       ActiveRecord::Base.connection.execute <<~SQL
-        WITH ci_job_artifacts_with_row_number as (
+        WITH ci_job_artifacts_with_row_number as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
           SELECT job_id, id, ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY id ASC) as row_number
           FROM ci_job_artifacts
           WHERE (file_type = #{LICENSE_SCANNING_FILE_TYPE} OR file_type = #{LICENSE_MANAGEMENT_FILE_TYPE})

db/post_migrate/20200831065705_ensure_target_project_id_is_filled.rb

@@ -32,7 +32,7 @@ def up
       )
       MergeRequestMetrics.connection.execute <<-SQL
-        WITH target_project_id_and_metrics_id as (
+        WITH target_project_id_and_metrics_id as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
           #{query_for_cte.to_sql}
         )
         UPDATE #{MergeRequestMetrics.connection.quote_table_name(MergeRequestMetrics.table_name)}

db/post_migrate/20210105030125_cleanup_projects_with_bad_has_external_wiki_data.rb

@@ -45,7 +45,7 @@ def update_projects_with_active_external_wikis
         .merge(Project.where(has_external_wiki: false).where(pending_delete: false).where(archived: false))
       execute(<<~SQL)
-      WITH project_ids_to_update (id) AS (
+      WITH project_ids_to_update (id) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
         #{scope_with_projects.to_sql}
       )
       UPDATE projects SET has_external_wiki = true WHERE id IN (SELECT id FROM project_ids_to_update)
@@ -75,7 +75,7 @@ def update_projects_without_active_external_wikis
     Project.where(index_where).each_batch(of: BATCH_SIZE) do |relation|
       relation_with_exists_query = relation.where('NOT EXISTS (?)', services_sub_query)
       execute(<<~SQL)
-      WITH project_ids_to_update (id) AS (
+      WITH project_ids_to_update (id) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
         #{relation_with_exists_query.select(:id).to_sql}
       )
       UPDATE projects SET has_external_wiki = false WHERE id IN (SELECT id FROM project_ids_to_update)

db/post_migrate/20210210221006_cleanup_projects_with_bad_has_external_issue_tracker_data.rb

@@ -44,7 +44,7 @@ def update_projects_with_active_external_issue_trackers
         .merge(Project.where(has_external_issue_tracker: false).where(pending_delete: false))
       execute(<<~SQL)
-      WITH project_ids_to_update (id) AS (
+      WITH project_ids_to_update (id) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
         #{scope_with_projects.to_sql}
       )
       UPDATE projects SET has_external_issue_tracker = true WHERE id IN (SELECT id FROM project_ids_to_update)
@@ -71,7 +71,7 @@ def update_projects_without_active_external_issue_trackers
     Project.where(index_where).each_batch(of: BATCH_SIZE) do |relation|
       relation_with_exists_query = relation.where('NOT EXISTS (?)', services_sub_query)
       execute(<<~SQL)
-      WITH project_ids_to_update (id) AS (
+      WITH project_ids_to_update (id) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
         #{relation_with_exists_query.select(:id).to_sql}
       )
       UPDATE projects SET has_external_issue_tracker = false WHERE id IN (SELECT id FROM project_ids_to_update)

ee/db/geo/migrate/20170906174622_remove_duplicates_from_project_registry.rb

@@ -7,7 +7,7 @@ class RemoveDuplicatesFromProjectRegistry < ActiveRecord::Migration[4.2]
   def up
     execute <<-SQL
-      WITH good_rows AS (
+      WITH good_rows AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
         SELECT project_id, MAX(id) as max_id
         FROM project_registry
         GROUP BY project_id

ee/lib/ee/gitlab/background_migration/fix_orphan_promoted_issues.rb

@@ -11,14 +11,14 @@ module FixOrphanPromotedIssues
         override :perform
         def perform(note_id)
           ActiveRecord::Base.connection.execute <<~SQL
-            WITH promotion_notes AS (
+            WITH promotion_notes AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
               SELECT noteable_id, note as promotion_note, projects.namespace_id as epic_group_id FROM notes
               INNER JOIN projects ON notes.project_id = projects.id
               WHERE notes.noteable_type = 'Issue'
               AND notes.system IS TRUE
               AND notes.note like 'promoted to epic%'
               AND notes.id = #{Integer(note_id)}
-            ), promoted_epics AS (
+            ), promoted_epics AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
               SELECT epics.id as promoted_epic_id, promotion_notes.noteable_id as issue_id FROM epics
               INNER JOIN promotion_notes on epics.group_id = promotion_notes.epic_group_id
               WHERE concat('promoted to epic &', epics.iid) = promotion_notes.promotion_note

lib/gitlab/background_migration/backfill_design_internal_ids.rb

@@ -97,7 +97,7 @@ def perform(relation)
         ActiveRecord::Base.connection.execute <<~SQL
           WITH
-          starting_iids(project_id, iid) as (
+          starting_iids(project_id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}(
             SELECT project_id, MAX(COALESCE(iid, 0))
             FROM #{table}
             WHERE project_id BETWEEN #{start_id} AND #{end_id}

lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb

@@ -8,7 +8,7 @@ def perform(*project_ids)
         updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id)
         Project.connection.execute <<-SQL
-          WITH repository_storage_cte as (
+          WITH repository_storage_cte as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
             #{updated_repository_storages.to_sql}
           )
           UPDATE projects

lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb

@@ -8,7 +8,7 @@ class CopyMergeRequestTargetProjectToMergeRequestMetrics
       def perform(start_id, stop_id)
         ActiveRecord::Base.connection.execute <<~SQL
-          WITH merge_requests_batch AS (
+          WITH merge_requests_batch AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
             SELECT id, target_project_id
             FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)}
           )

lib/gitlab/background_migration/fix_projects_without_project_feature.rb

@@ -22,7 +22,7 @@ def create_missing!(from_id, to_id)
       def sql(from_id, to_id)
         <<~SQL
-          WITH created_records AS (
+          WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
             INSERT INTO project_features (
               project_id,
               merge_requests_access_level,

lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb

@@ -136,7 +136,7 @@ def update_inconsistent(from_id, to_id)
       # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT
       def create_sql(from_id, to_id)
         <<~SQL
-          WITH created_records AS (
+          WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
             INSERT INTO services (project_id, #{DEFAULTS.keys.map { |key| %("#{key}")}.join(',')}, created_at, updated_at)
             #{select_insert_values_sql(from_id, to_id)}
             RETURNING *
@@ -149,7 +149,7 @@ def create_sql(from_id, to_id)
       # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT
       def update_sql(from_id, to_id)
         <<~SQL
-          WITH updated_records AS (
+          WITH updated_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
             UPDATE services SET active = TRUE
             WHERE services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)} AND services.properties = '{}' AND services.type = '#{Migratable::PrometheusService.type}'
             AND #{group_cluster_condition(from_id, to_id)} AND services.active = FALSE