Merge branch...

Merge branch 'philipcunningham-add-relationship-between-builds-and-dast-profiles-216514' into 'master'

Associate Build with DAST Site and Scanner profile [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!63362

app/services/ci/retry_build_service.rb

@@ -10,6 +10,10 @@ def self.clone_accessors
          resource_group scheduling_type].freeze
     end
+    def self.extra_accessors
+      []
+    end
+
     def execute(build)
       build.ensure_scheduling_type!

db/migrate/20210604032738_create_dast_site_profiles_builds.rb

+# frozen_string_literal: true
+
+class CreateDastSiteProfilesBuilds < ActiveRecord::Migration[6.1]
+  def up
+    table_comment = { owner: 'group::dynamic analysis', description: 'Join table between DAST Site Profiles and CI Builds' }
+
+    create_table :dast_site_profiles_builds, primary_key: [:dast_site_profile_id, :ci_build_id], comment: table_comment.to_json do |t|
+      t.bigint :dast_site_profile_id, null: false
+      t.bigint :ci_build_id, null: false
+
+      t.index :ci_build_id, unique: true, name: :dast_site_profiles_builds_on_ci_build_id
+    end
+  end
+
+  def down
+    drop_table :dast_site_profiles_builds
+  end
+end

db/migrate/20210604034158_add_ci_build_id_fk_to_dast_site_profiles_builds.rb

+# frozen_string_literal: true
+
+class AddCiBuildIdFkToDastSiteProfilesBuilds < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :dast_site_profiles_builds, :ci_builds, column: :ci_build_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :dast_site_profiles_builds, column: :ci_build_id
+    end
+  end
+end

db/migrate/20210604034354_add_dast_site_profile_id_fk_to_dast_site_profiles_builds.rb

+# frozen_string_literal: true
+
+class AddDastSiteProfileIdFkToDastSiteProfilesBuilds < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :dast_site_profiles_builds, :dast_site_profiles, column: :dast_site_profile_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :dast_site_profiles_builds, column: :dast_site_profile_id
+    end
+  end
+end

db/migrate/20210604051330_create_dast_scanner_profiles_builds.rb

+# frozen_string_literal: true
+
+class CreateDastScannerProfilesBuilds < ActiveRecord::Migration[6.1]
+  def up
+    table_comment = { owner: 'group::dynamic analysis', description: 'Join table between DAST Scanner Profiles and CI Builds' }
+
+    create_table :dast_scanner_profiles_builds, primary_key: [:dast_scanner_profile_id, :ci_build_id], comment: table_comment.to_json do |t|
+      t.bigint :dast_scanner_profile_id, null: false
+      t.bigint :ci_build_id, null: false
+
+      t.index :ci_build_id, unique: true, name: :dast_scanner_profiles_builds_on_ci_build_id
+    end
+  end
+
+  def down
+    drop_table :dast_scanner_profiles_builds
+  end
+end

db/migrate/20210604051742_add_ci_build_id_fk_to_dast_scanner_profiles_builds.rb

+# frozen_string_literal: true
+
+class AddCiBuildIdFkToDastScannerProfilesBuilds < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :dast_scanner_profiles_builds, :ci_builds, column: :ci_build_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :dast_scanner_profiles_builds, column: :ci_build_id
+    end
+  end
+end

db/migrate/20210604051917_add_dast_scanner_profile_id_fk_to_dast_scanner_profiles_builds.rb

+# frozen_string_literal: true
+
+class AddDastScannerProfileIdFkToDastScannerProfilesBuilds < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :dast_scanner_profiles_builds, :dast_scanner_profiles, column: :dast_scanner_profile_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :dast_scanner_profiles_builds, column: :dast_scanner_profile_id
+    end
+  end
+end

db/schema_migrations/20210604032738

+fa373e98739d57d829273cfa9246137e2c151be67e97183c1dcdb288150aaeb5
\ No newline at end of file

db/schema_migrations/20210604034158

+c7cf4aad7637d793d1ace8fee02111bc9b0d2eea09efadb0fd616bc5c5e5550c
\ No newline at end of file

db/schema_migrations/20210604034354

+da868be7c8edefc462110b5b36415870cc0c7c59dba1e3d514348011a9e70642
\ No newline at end of file

db/schema_migrations/20210604051330

+2d025932dca7a407968e14872ce053461e69550098ca089d4e6ece323d240927
\ No newline at end of file

db/schema_migrations/20210604051742

+7529373266b6c9b179367d5fa8775f5e2ad600008957b3a821d689aec70c7407
\ No newline at end of file

db/schema_migrations/20210604051917

+3818094a4470ff7d0c105c000655dac4205e8265f78df638df0e2ef3dc6deaf3
\ No newline at end of file

db/structure.sql

@@ -12047,6 +12047,13 @@ CREATE TABLE dast_scanner_profiles (
     CONSTRAINT check_568568fabf CHECK ((char_length(name) <= 255))
 );
+CREATE TABLE dast_scanner_profiles_builds (
+    dast_scanner_profile_id bigint NOT NULL,
+    ci_build_id bigint NOT NULL
+);
+
+COMMENT ON TABLE dast_scanner_profiles_builds IS '{"owner":"group::dynamic analysis","description":"Join table between DAST Scanner Profiles and CI Builds"}';
+
 CREATE SEQUENCE dast_scanner_profiles_id_seq
     START WITH 1
     INCREMENT BY 1
@@ -12102,6 +12109,13 @@ CREATE TABLE dast_site_profiles (
     CONSTRAINT check_f22f18002a CHECK ((char_length(auth_username) <= 255))
 );
+CREATE TABLE dast_site_profiles_builds (
+    dast_site_profile_id bigint NOT NULL,
+    ci_build_id bigint NOT NULL
+);
+
+COMMENT ON TABLE dast_site_profiles_builds IS '{"owner":"group::dynamic analysis","description":"Join table between DAST Site Profiles and CI Builds"}';
+
 CREATE SEQUENCE dast_site_profiles_id_seq
     START WITH 1
     INCREMENT BY 1
@@ -21092,12 +21106,18 @@ ALTER TABLE ONLY dast_profiles_pipelines
 ALTER TABLE ONLY dast_profiles
     ADD CONSTRAINT dast_profiles_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY dast_scanner_profiles_builds
+    ADD CONSTRAINT dast_scanner_profiles_builds_pkey PRIMARY KEY (dast_scanner_profile_id, ci_build_id);
+
 ALTER TABLE ONLY dast_scanner_profiles
     ADD CONSTRAINT dast_scanner_profiles_pkey PRIMARY KEY (id);
 ALTER TABLE ONLY dast_site_profile_secret_variables
     ADD CONSTRAINT dast_site_profile_secret_variables_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY dast_site_profiles_builds
+    ADD CONSTRAINT dast_site_profiles_builds_pkey PRIMARY KEY (dast_site_profile_id, ci_build_id);
+
 ALTER TABLE ONLY dast_site_profiles_pipelines
     ADD CONSTRAINT dast_site_profiles_pipelines_pkey PRIMARY KEY (dast_site_profile_id, ci_pipeline_id);
@@ -22343,6 +22363,10 @@ CREATE INDEX commit_id_and_note_id_index ON commit_user_mentions USING btree (co
 CREATE INDEX composer_cache_files_index_on_deleted_at ON packages_composer_cache_files USING btree (delete_at, id);
+CREATE UNIQUE INDEX dast_scanner_profiles_builds_on_ci_build_id ON dast_scanner_profiles_builds USING btree (ci_build_id);
+
+CREATE UNIQUE INDEX dast_site_profiles_builds_on_ci_build_id ON dast_site_profiles_builds USING btree (ci_build_id);
+
 CREATE UNIQUE INDEX design_management_designs_versions_uniqueness ON design_management_designs_versions USING btree (design_id, version_id);
 CREATE INDEX design_user_mentions_on_design_id_and_note_id_index ON design_user_mentions USING btree (design_id, note_id);
@@ -25705,6 +25729,9 @@ ALTER TABLE ONLY vulnerability_feedback
 ALTER TABLE ONLY deploy_keys_projects
     ADD CONSTRAINT fk_58a901ca7e FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+ALTER TABLE ONLY dast_scanner_profiles_builds
+    ADD CONSTRAINT fk_5d46286ad3 FOREIGN KEY (dast_scanner_profile_id) REFERENCES dast_scanner_profiles(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY issue_assignees
     ADD CONSTRAINT fk_5e0c8d9154 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
@@ -25864,6 +25891,9 @@ ALTER TABLE ONLY ci_pipeline_schedules
 ALTER TABLE ONLY todos
     ADD CONSTRAINT fk_91d1f47b13 FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE CASCADE;
+ALTER TABLE ONLY dast_site_profiles_builds
+    ADD CONSTRAINT fk_94e80df60e FOREIGN KEY (dast_site_profile_id) REFERENCES dast_site_profiles(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY vulnerability_feedback
     ADD CONSTRAINT fk_94f7c8a81e FOREIGN KEY (comment_author_id) REFERENCES users(id) ON DELETE SET NULL;
@@ -25927,6 +25957,9 @@ ALTER TABLE ONLY ci_builds
 ALTER TABLE ONLY ci_pipelines
     ADD CONSTRAINT fk_a23be95014 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE;
+ALTER TABLE ONLY dast_site_profiles_builds
+    ADD CONSTRAINT fk_a325505e99 FOREIGN KEY (ci_build_id) REFERENCES ci_builds(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY bulk_import_entities
     ADD CONSTRAINT fk_a44ff95be5 FOREIGN KEY (parent_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE;
@@ -26137,6 +26170,9 @@ ALTER TABLE ONLY gitlab_subscriptions
 ALTER TABLE ONLY ci_triggers
     ADD CONSTRAINT fk_e3e63f966e FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+ALTER TABLE ONLY dast_scanner_profiles_builds
+    ADD CONSTRAINT fk_e4c49200f8 FOREIGN KEY (ci_build_id) REFERENCES ci_builds(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY merge_requests
     ADD CONSTRAINT fk_e719a85f8a FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL;

ee/app/models/concerns/app_sec/dast/buildable.rb

+# frozen_string_literal: true
+
+module AppSec
+  module Dast
+    module Buildable
+      extend ::ActiveSupport::Concern
+
+      included do
+        extend SuppressCompositePrimaryKeyWarning
+
+        validate :project_ids_match
+      end
+
+      def project_ids_match
+        return if ci_build.nil? || profile.nil?
+
+        unless ci_build.project_id == profile.project_id
+          errors.add(:ci_build_id, "project_id must match #{profile.class.underscore}.project_id")
+        end
+      end
+    end
+  end
+end

ee/app/models/dast/scanner_profiles_build.rb

+# frozen_string_literal: true
+
+module Dast
+  class ScannerProfilesBuild < ApplicationRecord
+    include AppSec::Dast::Buildable
+
+    self.table_name = 'dast_scanner_profiles_builds'
+
+    belongs_to :ci_build, class_name: 'Ci::Build', optional: false, inverse_of: :dast_scanner_profiles_build
+    belongs_to :dast_scanner_profile, class_name: 'DastScannerProfile', optional: false, inverse_of: :dast_scanner_profiles_builds
+
+    validates :ci_build_id, :dast_scanner_profile_id, presence: true
+
+    alias_attribute :profile, :dast_scanner_profile
+  end
+end

ee/app/models/dast/site_profiles_build.rb

+# frozen_string_literal: true
+
+module Dast
+  class SiteProfilesBuild < ApplicationRecord
+    include AppSec::Dast::Buildable
+
+    self.table_name = 'dast_site_profiles_builds'
+
+    belongs_to :ci_build, class_name: 'Ci::Build', optional: false, inverse_of: :dast_site_profiles_build
+    belongs_to :dast_site_profile, class_name: 'DastSiteProfile', optional: false, inverse_of: :dast_site_profiles_builds
+
+    validates :ci_build_id, :dast_site_profile_id, presence: true
+
+    alias_attribute :profile, :dast_site_profile
+  end
+end

ee/app/models/dast_scanner_profile.rb

@@ -3,6 +3,9 @@
 class DastScannerProfile < ApplicationRecord
   belongs_to :project
+  has_many :dast_scanner_profiles_builds, class_name: 'Dast::ScannerProfilesBuild', foreign_key: :dast_scanner_profile_id, inverse_of: :dast_scanner_profile
+  has_many :ci_builds, class_name: 'Ci::Build', through: :dast_scanner_profiles_builds
+
   validates :project_id, presence: true
   validates :name, length: { maximum: 255 }, uniqueness: { scope: :project_id }, presence: true

ee/app/models/dast_site_profile.rb

@@ -9,6 +9,9 @@ class DastSiteProfile < ApplicationRecord
   has_many :dast_site_profiles_pipelines, class_name: 'Dast::SiteProfilesPipeline', foreign_key: :dast_site_profile_id, inverse_of: :dast_site_profile
   has_many :ci_pipelines, class_name: 'Ci::Pipeline', through: :dast_site_profiles_pipelines
+  has_many :dast_site_profiles_builds, class_name: 'Dast::SiteProfilesBuild', foreign_key: :dast_site_profile_id, inverse_of: :dast_site_profile
+  has_many :ci_builds, class_name: 'Ci::Build', through: :dast_site_profiles_builds
+
   validates :excluded_urls, length: { maximum: 25 }
   validates :auth_url, addressable_url: true, length: { maximum: 1024 }, allow_nil: true
   validates :auth_username_field, :auth_password_field, :auth_username, length: { maximum: 255 }

ee/app/models/ee/ci/build.rb

@@ -31,6 +31,12 @@ module Build
         has_many :security_scans, class_name: 'Security::Scan'
+        has_one :dast_site_profiles_build, class_name: 'Dast::SiteProfilesBuild', foreign_key: :ci_build_id, inverse_of: :ci_build
+        has_one :dast_site_profile, class_name: 'DastSiteProfile', through: :dast_site_profiles_build
+
+        has_one :dast_scanner_profiles_build, class_name: 'Dast::ScannerProfilesBuild', foreign_key: :ci_build_id, inverse_of: :ci_build
+        has_one :dast_scanner_profile, class_name: 'DastScannerProfile', through: :dast_scanner_profiles_build
+
         after_commit :track_ci_secrets_management_usage, on: :create
         delegate :service_specification, to: :runner_session, allow_nil: true
@@ -47,12 +53,8 @@ module Build
       def variables
         strong_memoize(:variables) do
           super.tap do |collection|
-            if pipeline.triggered_for_ondemand_dast_scan?
-              # Subject to change. Please see gitlab-org/gitlab#330950 for more info.
-              profile = pipeline.dast_profile || pipeline.dast_site_profile
-
-              collection.concat(profile.secret_ci_variables(pipeline.user))
-            end
+            collection.concat(dast_on_demand_variables)
+            collection.concat(dast_configuration_variables)
           end
         end
       end
@@ -169,6 +171,33 @@ def variables_hash
         end
       end
+      def dast_on_demand_variables
+        ::Gitlab::Ci::Variables::Collection.new.tap do |collection|
+          break collection unless pipeline.triggered_for_ondemand_dast_scan?
+
+          # Subject to change. Please see gitlab-org/gitlab#330950 for more info.
+          profile = pipeline.dast_profile || pipeline.dast_site_profile
+
+          collection.concat(profile.secret_ci_variables(pipeline.user))
+        end
+      end
+
+      def dast_configuration_variables
+        ::Gitlab::Ci::Variables::Collection.new.tap do |collection|
+          break collection unless ::Feature.enabled?(:dast_configuration_ui, project)
+          break collection unless (dast_configuration = options[:dast_configuration])
+
+          if dast_configuration[:site_profile] && dast_site_profile
+            collection.concat(dast_site_profile.ci_variables)
+            collection.concat(dast_site_profile.secret_ci_variables(user))
+          end
+
+          if dast_configuration[:scanner_profile] && dast_scanner_profile
+            collection.concat(dast_scanner_profile.ci_variables)
+          end
+        end
+      end
+
       def parse_security_artifact_blob(security_report, blob)
         report_clone = security_report.clone_as_blank
         signatures_enabled = ::Feature.enabled?(:vulnerability_finding_tracking_signatures, project) && project.licensed_feature_available?(:vulnerability_finding_signatures)