Merge branch '423036-add-table-instance-gcp' into 'master'

Added table and migrations for instance level GCP config

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/130079



Merged-by: Huzaifa Iftikhar <hiftikhar@gitlab.com>
Approved-by: Tianwen Chen <tchen@gitlab.com>
Approved-by: Huzaifa Iftikhar <hiftikhar@gitlab.com>
Approved-by: Omar Qunsul <oqunsul@gitlab.com>
Reviewed-by: A Browne <abrowne@gitlab.com>
Reviewed-by: Sam Figueroa <sfigueroa@gitlab.com>
Reviewed-by: Omar Qunsul <oqunsul@gitlab.com>
Reviewed-by: Hitesh Raghuvanshi <hraghuvanshi@gitlab.com>
Reviewed-by: Tianwen Chen <tchen@gitlab.com>
Co-authored-by: Hitesh Raghuvanshi <hraghuvanshi@gitlab.com>

db/docs/audit_events_instance_google_cloud_logging_configurations.yml

+---
+table_name: audit_events_instance_google_cloud_logging_configurations
+classes:
+  - AuditEvents::Instance::GoogleCloudLoggingConfiguration
+feature_categories:
+  - audit_events
+description: Stores Instance level Google Cloud Logging configurations associated with IAM service accounts, used for generating access tokens.
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/423036
+milestone: '16.4'
+gitlab_schema: gitlab_main

db/migrate/20230823132142_create_instance_google_cloud_logging_configurations.rb

+# frozen_string_literal: true
+
+class CreateInstanceGoogleCloudLoggingConfigurations < Gitlab::Database::Migration[2.1]
+  enable_lock_retries!
+
+  UNIQUE_INDEX_NAME = "unique_instance_google_cloud_logging_configurations"
+  UNIQUE_CONFIG_NAME_INDEX = "unique_instance_google_cloud_logging_configurations_name"
+
+  def change
+    create_table :audit_events_instance_google_cloud_logging_configurations do |t|
+      t.timestamps_with_timezone null: false
+      t.text :google_project_id_name, null: false, limit: 30
+      t.text :client_email, null: false, limit: 254
+      t.text :log_id_name, default: "audit_events", limit: 511
+      t.text :name, null: false, limit: 72
+      t.binary :encrypted_private_key, null: false
+      t.binary :encrypted_private_key_iv, null: false
+
+      t.index [:google_project_id_name, :log_id_name], unique: true, name: UNIQUE_INDEX_NAME
+      t.index :name, unique: true, name: UNIQUE_CONFIG_NAME_INDEX
+    end
+  end
+end

db/schema_migrations/20230823132142

+438ce9b705e575c139724a2ad81e799bdda63b9428b4cf3bd21b9e47df36ece7
\ No newline at end of file

db/structure.sql

@@ -12379,6 +12379,31 @@ CREATE SEQUENCE audit_events_instance_external_audit_event_destinations_id_seq
 ALTER SEQUENCE audit_events_instance_external_audit_event_destinations_id_seq OWNED BY audit_events_instance_external_audit_event_destinations.id;
+CREATE TABLE audit_events_instance_google_cloud_logging_configurations (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    google_project_id_name text NOT NULL,
+    client_email text NOT NULL,
+    log_id_name text DEFAULT 'audit_events'::text,
+    name text NOT NULL,
+    encrypted_private_key bytea NOT NULL,
+    encrypted_private_key_iv bytea NOT NULL,
+    CONSTRAINT check_0da5c76c49 CHECK ((char_length(client_email) <= 254)),
+    CONSTRAINT check_74fd943192 CHECK ((char_length(log_id_name) <= 511)),
+    CONSTRAINT check_ab65f57721 CHECK ((char_length(google_project_id_name) <= 30)),
+    CONSTRAINT check_ac42ad3ca2 CHECK ((char_length(name) <= 72))
+);
+
+CREATE SEQUENCE audit_events_instance_google_cloud_logging_configuration_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE audit_events_instance_google_cloud_logging_configuration_id_seq OWNED BY audit_events_instance_google_cloud_logging_configurations.id;
+
 CREATE TABLE audit_events_streaming_event_type_filters (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -25422,6 +25447,8 @@ ALTER TABLE ONLY audit_events_google_cloud_logging_configurations ALTER COLUMN i
 ALTER TABLE ONLY audit_events_instance_external_audit_event_destinations ALTER COLUMN id SET DEFAULT nextval('audit_events_instance_external_audit_event_destinations_id_seq'::regclass);
+ALTER TABLE ONLY audit_events_instance_google_cloud_logging_configurations ALTER COLUMN id SET DEFAULT nextval('audit_events_instance_google_cloud_logging_configuration_id_seq'::regclass);
+
 ALTER TABLE ONLY audit_events_streaming_event_type_filters ALTER COLUMN id SET DEFAULT nextval('audit_events_streaming_event_type_filters_id_seq'::regclass);
 ALTER TABLE ONLY audit_events_streaming_headers ALTER COLUMN id SET DEFAULT nextval('audit_events_streaming_headers_id_seq'::regclass);
@@ -27215,6 +27242,9 @@ ALTER TABLE ONLY audit_events_google_cloud_logging_configurations
 ALTER TABLE ONLY audit_events_instance_external_audit_event_destinations
     ADD CONSTRAINT audit_events_instance_external_audit_event_destinations_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY audit_events_instance_google_cloud_logging_configurations
+    ADD CONSTRAINT audit_events_instance_google_cloud_logging_configurations_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY audit_events
     ADD CONSTRAINT audit_events_pkey PRIMARY KEY (id, created_at);
@@ -34328,6 +34358,10 @@ CREATE UNIQUE INDEX unique_index_sysaccess_ms_access_tokens_on_sysaccess_ms_app_
 CREATE UNIQUE INDEX unique_instance_audit_event_destination_name ON audit_events_instance_external_audit_event_destinations USING btree (name);
+CREATE UNIQUE INDEX unique_instance_google_cloud_logging_configurations ON audit_events_instance_google_cloud_logging_configurations USING btree (google_project_id_name, log_id_name);
+
+CREATE UNIQUE INDEX unique_instance_google_cloud_logging_configurations_name ON audit_events_instance_google_cloud_logging_configurations USING btree (name);
+
 CREATE UNIQUE INDEX unique_merge_request_diff_llm_summaries_on_mr_diff_id ON merge_request_diff_llm_summaries USING btree (merge_request_diff_id);
 CREATE UNIQUE INDEX unique_merge_request_metrics_by_merge_request_id ON merge_request_metrics USING btree (merge_request_id);

ee/app/models/audit_events/google_cloud_logging_configuration.rb

@@ -4,54 +4,21 @@ module AuditEvents
   class GoogleCloudLoggingConfiguration < ApplicationRecord
     include Limitable
     include ExternallyCommonDestinationable
+    include GcpExternallyDestinationable
     self.limit_name = 'google_cloud_logging_configurations'
     self.limit_scope = :group
     self.table_name = 'audit_events_google_cloud_logging_configurations'
-    GOOGLE_PROJECT_ID_NAME_REGEX = %r{\A[a-z][a-z0-9-]*[a-z0-9]\z}
-    LOG_ID_NAME_REGEX = %r{\A[\w/.-]+\z}
-
-    DEFAULT_LOG_ID_NAME = "audit_events"
-
-    attribute :log_id_name, :string, default: DEFAULT_LOG_ID_NAME
-
     belongs_to :group, class_name: '::Group', foreign_key: 'namespace_id',
       inverse_of: :google_cloud_logging_configurations
-    validates :google_project_id_name, presence: true,
-      format: { with: GOOGLE_PROJECT_ID_NAME_REGEX,
-                message: 'must only contain lowercase letters, digits, or hyphens, ' \
-                         'and must start and end with a letter or digit' },
-      length: { in: 6..30 },
-      uniqueness: { scope: [:namespace_id, :log_id_name] }
-    validates :log_id_name, presence: true,
-      format: { with: LOG_ID_NAME_REGEX,
-                message: 'must only contain letters, digits, forward-slash, underscore, hyphen or period' },
-      length: { maximum: 511 }
-
-    validates :client_email, presence: true, format: { with: URI::MailTo::EMAIL_REGEXP }, length: { maximum: 254 }
-    validates :private_key, presence: true
+    validates :google_project_id_name, uniqueness: { scope: [:namespace_id, :log_id_name] }
     validates :name, uniqueness: { scope: :namespace_id }
-    attr_encrypted :private_key,
-      mode: :per_attribute_iv,
-      key: Settings.attr_encrypted_db_key_base_32,
-      algorithm: 'aes-256-gcm',
-      encode: false,
-      encode_iv: false
-
     validate :root_level_group?
-    def allowed_to_stream?(*)
-      true
-    end
-
-    def full_log_path
-      "projects/#{google_project_id_name}/logs/#{log_id_name}"
-    end
-
     private
     def root_level_group?

ee/app/models/audit_events/instance/google_cloud_logging_configuration.rb

+# frozen_string_literal: true
+
+module AuditEvents
+  module Instance
+    class GoogleCloudLoggingConfiguration < ApplicationRecord
+      include Limitable
+      include ExternallyCommonDestinationable
+      include GcpExternallyDestinationable
+
+      self.limit_name = 'google_cloud_logging_configurations'
+      self.limit_scope = Limitable::GLOBAL_SCOPE
+      self.table_name = 'audit_events_instance_google_cloud_logging_configurations'
+
+      validates :log_id_name, uniqueness: { scope: :google_project_id_name }
+      validates :name, uniqueness: true
+    end
+  end
+end

ee/app/models/concerns/audit_events/gcp_externally_destinationable.rb

+# frozen_string_literal: true
+
+module AuditEvents
+  module GcpExternallyDestinationable
+    extend ActiveSupport::Concern
+
+    GOOGLE_PROJECT_ID_NAME_REGEX = %r{\A[a-z][a-z0-9-]*[a-z0-9]\z}
+    LOG_ID_NAME_REGEX = %r{\A[\w/.-]+\z}
+
+    DEFAULT_LOG_ID_NAME = "audit_events"
+
+    included do
+      attribute :log_id_name, :string, default: DEFAULT_LOG_ID_NAME
+
+      validates :log_id_name, presence: true,
+        format: { with: LOG_ID_NAME_REGEX,
+                  message: ->(_object, _data) {
+                    _('must only contain letters, digits, forward-slash, underscore, hyphen or period')
+                  } },
+        length: { maximum: 511 }
+
+      validates :client_email, presence: true, format: { with: URI::MailTo::EMAIL_REGEXP }, length: { maximum: 254 }
+      validates :private_key, presence: true
+
+      validates :google_project_id_name, presence: true,
+        format: {
+          with: GOOGLE_PROJECT_ID_NAME_REGEX,
+          message: 'must only contain lowercase letters, digits, or hyphens, ' \
+                   'and must start and end with a letter or digit'
+        },
+        length: { in: 6..30 }
+
+      attr_encrypted :private_key,
+        mode: :per_attribute_iv,
+        key: Settings.attr_encrypted_db_key_base_32,
+        algorithm: 'aes-256-gcm',
+        encode: false,
+        encode_iv: false
+
+      def allowed_to_stream?(*)
+        true
+      end
+
+      def full_log_path
+        "projects/#{google_project_id_name}/logs/#{log_id_name}"
+      end
+    end
+  end
+end

ee/spec/factories/audit_events/instance_google_cloud_logging_configurations.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :instance_google_cloud_logging_configuration,
+    class: 'AuditEvents::Instance::GoogleCloudLoggingConfiguration' do
+    sequence :google_project_id_name do |i|
+      "#{FFaker::Lorem.word.downcase}-#{SecureRandom.hex(4)}-#{i}"
+    end
+    client_email { FFaker::Internet.safe_email }
+    log_id_name { 'audit_events' }
+    private_key { OpenSSL::PKey::RSA.new(4096).to_pem }
+  end
+end

ee/spec/models/audit_events/instance/google_cloud_logging_configuration_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe AuditEvents::Instance::GoogleCloudLoggingConfiguration, feature_category: :audit_events do
+  subject(:instance_google_cloud_logging_config) { build(:instance_google_cloud_logging_configuration) }
+
+  describe 'Validations' do
+    it { is_expected.to validate_presence_of(:google_project_id_name) }
+    it { is_expected.to validate_presence_of(:client_email) }
+    it { is_expected.to validate_presence_of(:log_id_name) }
+    it { is_expected.to validate_presence_of(:private_key) }
+
+    it { is_expected.to validate_length_of(:google_project_id_name).is_at_least(6).is_at_most(30) }
+    it { is_expected.to validate_length_of(:client_email).is_at_most(254) }
+    it { is_expected.to validate_length_of(:log_id_name).is_at_most(511) }
+    it { is_expected.to validate_length_of(:name).is_at_most(72) }
+
+    it { is_expected.to allow_value('valid-project-id').for(:google_project_id_name) }
+    it { is_expected.not_to allow_value('invalid_project_id').for(:google_project_id_name) }
+    it { is_expected.not_to allow_value('invalid-project-id-').for(:google_project_id_name) }
+    it { is_expected.not_to allow_value('Invalid-project-id').for(:google_project_id_name) }
+    it { is_expected.not_to allow_value('1-invalid-project-id').for(:google_project_id_name) }
+
+    it { is_expected.to allow_value('valid@example.com').for(:client_email) }
+    it { is_expected.to allow_value('valid@example.org').for(:client_email) }
+    it { is_expected.to allow_value('valid@example.co.uk').for(:client_email) }
+    it { is_expected.to allow_value('valid_email+mail@mail.com').for(:client_email) }
+    it { is_expected.not_to allow_value('invalid_email').for(:client_email) }
+    it { is_expected.not_to allow_value('invalid@.com').for(:client_email) }
+    it { is_expected.not_to allow_value('invalid..com').for(:client_email) }
+
+    it { is_expected.to allow_value('audit_events').for(:log_id_name) }
+    it { is_expected.to allow_value('audit-events').for(:log_id_name) }
+    it { is_expected.to allow_value('audit.events').for(:log_id_name) }
+    it { is_expected.to allow_value('AUDIT_EVENTS').for(:log_id_name) }
+    it { is_expected.to allow_value('audit_events/123').for(:log_id_name) }
+    it { is_expected.not_to allow_value('AUDIT_EVENT@').for(:log_id_name) }
+    it { is_expected.not_to allow_value('AUDIT_EVENT$').for(:log_id_name) }
+    it { is_expected.not_to allow_value('#AUDIT_EVENT').for(:log_id_name) }
+    it { is_expected.not_to allow_value('%audit_events/123').for(:log_id_name) }
+
+    context 'when the same google_project_id_name for the same log_id_name exists' do
+      let(:google_project_id_name) { 'valid-project-id' }
+      let(:log_id_name) { 'audit_events' }
+
+      before do
+        create(:instance_google_cloud_logging_configuration, google_project_id_name: google_project_id_name,
+          log_id_name: log_id_name)
+      end
+
+      it 'is not valid and adds an error message' do
+        config = build(:instance_google_cloud_logging_configuration, google_project_id_name: google_project_id_name,
+          log_id_name: log_id_name)
+        expect(config).not_to be_valid
+        expect(config.errors[:log_id_name]).to include('has already been taken')
+      end
+    end
+
+    it 'validates uniqueness of name' do
+      create(:instance_google_cloud_logging_configuration, name: 'Test Destination')
+      destination = build(:instance_google_cloud_logging_configuration, name: 'Test Destination')
+
+      expect(destination).not_to be_valid
+      expect(destination.errors.full_messages).to include('Name has already been taken')
+    end
+  end
+
+  describe 'default values' do
+    it "uses 'audit_events' as default value for log_id_name" do
+      expect(described_class.new.log_id_name).to eq('audit_events')
+    end
+  end
+
+  describe '#allowed_to_stream?' do
+    it 'always returns true' do
+      expect(instance_google_cloud_logging_config.allowed_to_stream?).to eq(true)
+    end
+  end
+
+  describe '#full_log_path' do
+    it 'returns the full log path for the google project' do
+      instance_google_cloud_logging_config.google_project_id_name = "test-project"
+      instance_google_cloud_logging_config.log_id_name = "test-log"
+
+      expect(instance_google_cloud_logging_config.full_log_path).to eq("projects/test-project/logs/test-log")
+    end
+  end
+
+  it_behaves_like 'includes Limitable concern' do
+    subject { build(:instance_google_cloud_logging_configuration) }
+  end
+
+  it_behaves_like 'includes ExternallyCommonDestinationable concern' do
+    let(:model_factory_name) { :instance_google_cloud_logging_configuration }
+  end
+end

locale/gitlab.pot

@@ -56789,6 +56789,9 @@ msgstr ""
 msgid "must not contain commonly used combinations of words and letters"
 msgstr ""
+msgid "must only contain letters, digits, forward-slash, underscore, hyphen or period"
+msgstr ""
+
 msgid "my-awesome-group"
 msgstr ""