Merge branch '405028-enable-sync' into 'master'

Enable sync with package metadata db by default

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/130176



Merged-by: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com>
Approved-by: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com>
Approved-by: Ahmed Hemdan <ahemdan@gitlab.com>
Reviewed-by: Grant Young <gyoung@gitlab.com>
Reviewed-by: Igor Frenkel <ifrenkel@gitlab.com>
Reviewed-by: Ahmed Hemdan <ahemdan@gitlab.com>
Co-authored-by: Tetiana Chupryna <tchupryna@gitlab.com>
Co-authored-by: Andrejs Cunskis <acunskis@gitlab.com>

db/migrate/20230823160533_update_package_metadata_sync_setting.rb

+# frozen_string_literal: true
+
+class UpdatePackageMetadataSyncSetting < Gitlab::Database::Migration[2.1]
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  class ApplicationSetting < MigrationRecord
+  end
+
+  FULLY_ENABLED_SYNC = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12].freeze
+
+  def up
+    application_setting = ApplicationSetting.last
+    return unless application_setting
+
+    # Check if the setting still has a default value and it wasn't updated manually by the admin
+    return unless application_setting.package_metadata_purl_types == []
+
+    # Update setting to enable all package types to sync
+    application_setting.update(package_metadata_purl_types: FULLY_ENABLED_SYNC)
+  end
+
+  def down
+    # no op
+  end
+end

db/migrate/20230830101525_update_default_value_pm.rb

+# frozen_string_literal: true
+
+class UpdateDefaultValuePm < Gitlab::Database::Migration[2.1]
+  disable_ddl_transaction!
+
+  FULLY_ENABLED_SYNC = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12].freeze
+
+  def change
+    change_column_default :application_settings, :package_metadata_purl_types, from: [], to: FULLY_ENABLED_SYNC
+  end
+end

db/schema_migrations/20230823160533

+1583a9581ec2905781f4a5bb7715c7da784d6772eb6b6d8ecb05cad53f13b8c9
\ No newline at end of file

db/schema_migrations/20230830101525

+02b0d2f9133db9378d2511144c4cff91a5d2ea9dce30eed371122dec342d547b
\ No newline at end of file

db/structure.sql

@@ -11913,7 +11913,7 @@ CREATE TABLE application_settings (
     encrypted_product_analytics_configurator_connection_string bytea,
     encrypted_product_analytics_configurator_connection_string_iv bytea,
     silent_mode_enabled boolean DEFAULT false NOT NULL,
-    package_metadata_purl_types smallint[] DEFAULT '{}'::smallint[],
+    package_metadata_purl_types smallint[] DEFAULT '{1,2,3,4,5,6,7,8,9,10,11,12}'::smallint[],
     ci_max_includes integer DEFAULT 150 NOT NULL,
     remember_me_enabled boolean DEFAULT true NOT NULL,
     encrypted_anthropic_api_key bytea,

doc/api/settings.md

@@ -481,6 +481,7 @@ listed in the descriptions of the relevant settings.
 | `pypi_package_requests_forwarding` **(PREMIUM ALL)**  | boolean | no                                   | Use pypi.org as a default remote repository when the package is not found in the GitLab Package Registry for PyPI. |
 | `outbound_local_requests_whitelist`      | array of strings | no                                   | Define a list of trusted domains or IP addresses to which local requests are allowed when local requests for webhooks and integrations are disabled.
 | `package_registry_allow_anyone_to_pull_option` | boolean    | no                                   | Enable to [allow anyone to pull from Package Registry](../user/packages/package_registry/index.md#allow-anyone-to-pull-from-package-registry) visible and changeable.
+| `package_metadata_purl_types` **(ULTIMATE SELF)** | array of integers | no                         | List of [package registry metadata to sync](../administration/settings/security_and_compliance.md#choose-package-registry-metadata-to-sync). See [the list](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/concerns/enums/package_metadata.rb#L5) of the available values.
 | `pages_domain_verification_enabled`       | boolean         | no                                   | Require users to prove ownership of custom domains. Domain verification is an essential security measure for public GitLab sites. Users are required to demonstrate they control a domain before it is enabled. |
 | `password_authentication_enabled_for_git` | boolean         | no                                   | Enable authentication for Git over HTTP(S) via a GitLab account password. Default is `true`. |
 | `password_authentication_enabled_for_web` | boolean         | no                                   | Enable authentication for the web interface via a GitLab account password. Default is `true`. |

ee/app/helpers/ee/application_settings_helper.rb

@@ -62,6 +62,7 @@ def visible_attributes
         :throttle_incident_management_notification_enabled,
         :throttle_incident_management_notification_per_period,
         :throttle_incident_management_notification_period_in_seconds,
+        :package_metadata_purl_types,
         :product_analytics_enabled,
         :product_analytics_data_collector_host,
         :product_analytics_configurator_connection_string,

ee/spec/requests/api/settings_spec.rb

@@ -27,12 +27,14 @@
       put api(path, admin, admin_mode: true),
         params: {
           help_text: 'Help text',
-          file_template_project_id: project.id
+          file_template_project_id: project.id,
+          package_metadata_purl_types: [1]
         }
       expect(response).to have_gitlab_http_status(:ok)
       expect(json_response['help_text']).to eq('Help text')
       expect(json_response['file_template_project_id']).to eq(project.id)
+      expect(json_response['package_metadata_purl_types']).to eq([1])
     end
     context 'elasticsearch settings' do

ee/spec/services/package_metadata/sync_service_spec.rb

@@ -181,15 +181,17 @@
     subject(:execute) { described_class.execute(data_type: data_type, lease: lease) }
+    before do
+      allow(PackageMetadata::StopSignal).to receive(:new)
+        .with(lease, described_class::MAX_LEASE_LENGTH, described_class::MAX_SYNC_DURATION)
+        .and_return(stop_signal)
+    end
+
     shared_examples_for 'it calls #execute for each enabled config' do
       let(:should_stop) { false }
       before do
-        stub_application_setting(package_metadata_purl_types: Enums::PackageMetadata.purl_types.values)
         stub_feature_flags(compressed_package_metadata_synchronization: false)
-        allow(PackageMetadata::StopSignal).to receive(:new)
-          .with(lease, described_class::MAX_LEASE_LENGTH, described_class::MAX_SYNC_DURATION)
-          .and_return(stop_signal)
       end
       specify do

qa/qa/ee/strategy.rb

@@ -15,6 +15,12 @@ def perform_before_hooks
         EE::Resource::License.fabricate! do |resource|
           resource.license = QA::Runtime::Env.ee_license
         end
+
+        unless QA::Runtime::Env.running_on_dot_com?
+          QA::Runtime::Logger.info("Disabling sync with External package metadata database")
+          QA::Runtime::ApplicationSettings.set_application_settings(package_metadata_purl_types: [12])
+        end
+
         QA::Page::Main::Menu.perform(&:sign_out_if_signed_in)
       end
     end

spec/migrations/db/migrate/20230823160533_update_package_metadata_sync_setting_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe UpdatePackageMetadataSyncSetting, feature_category: :software_composition_analysis do
+  let(:settings) { table(:application_settings) }
+  let(:fully_enabled_sync_setting) { [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] }
+
+  describe "#up" do
+    context 'with default value' do
+      let(:fully_disabled_sync) { [] }
+
+      it 'updates setting' do
+        settings.create!(package_metadata_purl_types: fully_disabled_sync)
+
+        migrate!
+
+        expect(ApplicationSetting.last.package_metadata_purl_types).to eq(fully_enabled_sync_setting)
+      end
+    end
+
+    context 'with custom value' do
+      let(:partially_enabled_sync) { [1, 2, 3, 4, 5] }
+
+      it 'does not change setting' do
+        settings.create!(package_metadata_purl_types: partially_enabled_sync)
+
+        migrate!
+
+        expect(ApplicationSetting.last.package_metadata_purl_types).to eq(partially_enabled_sync)
+      end
+    end
+  end
+end