Enable selective test execution for test-on-cng

Fix rule comment

Add selective parallel and inverse suites matching

Adjust comments

Fix reusable if rules

Correctly trigger full suite rule

Use separate rule for default full suite run

Add link to selective test execution mapping

.gitlab/ci/test-on-cng/main.gitlab-ci.yml

@@ -10,6 +10,50 @@ workflow:
   rules:
     - when: always
+.mr-code-patterns: &mr-code-patterns
+  if: $MR_CODE_PATTERNS == "true"
+
+# list of specific tests is used when it is small enough to run without knapsack parallelization
+.tests-set: &tests-set
+  if: $QA_TESTS != ""
+
+# pattern is used in case when specific tests are selected but the amount of tests requires multiple jobs
+.parallel-pattern-set: &parallel-pattern-set
+  if: $KNAPSACK_TEST_FILE_PATTERN != ""
+
+.rules:selective:
+  rules:
+    # Skip selective jobs if mr contains app code patterns
+    - <<: *mr-code-patterns
+      when: never
+    - <<: *parallel-pattern-set
+      when: never
+    - <<: *tests-set
+
+.rules:selective-parallel:
+  rules:
+    # Skip selective jobs if mr contains app code patterns
+    - <<: *mr-code-patterns
+      when: never
+    - <<: *tests-set
+      when: never
+    - <<: *parallel-pattern-set
+
+.rules:full:
+  rules:
+    # Run full test suite for application code changes
+    # TODO: Remove once proper dynamic code mapping is in place - https://gitlab.com/groups/gitlab-org/quality/quality-engineering/-/epics/47
+    - if: $MR_CODE_PATTERNS == "true"
+      variables:
+        QA_TESTS: ""
+        KNAPSACK_TEST_FILE_PATTERN: ""
+    # Run full suite when both selective execution variables are unpopulated
+    - if: $QA_TESTS == "" && $KNAPSACK_TEST_FILE_PATTERN == ""
+    - <<: *tests-set
+      when: never
+    - <<: *parallel-pattern-set
+      when: never
+
 .cng-test:
   image: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:git-${GIT_VERSION}-lfs-${LFS_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-kubectl-${KUBECTL_VERSION}-helm-${HELM_VERSION}-kind-${KIND_VERSION}"
   stage: test
@@ -40,9 +84,6 @@ workflow:
     CNG_FORCE_COLOR: "true"
     QA_RUN_IN_PARALLEL: "true"
     QA_PARALLEL_PROCESSES: 4
-    # disable selective test execution until pipeline setup is implemented to support it correctly
-    KNAPSACK_TEST_FILE_PATTERN: ""
-    QA_TESTS: ""
   before_script:
     - echo "SUITE_RAN=true" > "$QA_SUITE_STATUS_ENV_FILE"
     # save extra values to be available for after_script if created dynamically
@@ -121,23 +162,51 @@ download-knapsack-report:
 # ==========================================
 # Test stage
 # ==========================================
+
+# == main 'instance' test suite ==
 cng-instance:
   extends: .cng-test-parallel
   variables:
     DEPLOYMENT_TYPE: kind
     EXTRA_DEPLOY_VALUES: --env COVERBAND_ENABLED=${COVERBAND_ENABLED}
-# Test run against environment with minimum supported redis version defined in lib/system_check/app/redis_version_check.rb
+  rules:
+    - if: $QA_SUITES !~ /Test::Instance::All/
+      when: never
+    - !reference [.rules:full, rules]
+cng-instance-selective-parallel:
+  extends: .cng-test-parallel
+  variables:
+    DEPLOYMENT_TYPE: kind
+  parallel: 2
+  rules:
+    - if: $QA_SUITES !~ /Test::Instance::All/
+      when: never
+    - !reference [.rules:selective-parallel, rules]
+
+cng-instance-selective:
+  extends: .cng-test
+  variables:
+    DEPLOYMENT_TYPE: kind
+  rules:
+    - if: $QA_SUITES !~ /Test::Instance::All/
+      when: never
+    - !reference [.rules:selective, rules]
+
+# == minimal supported redis version ==
 cng-qa-min-redis-version:
   extends: .cng-test
   variables:
     DEPLOYMENT_TYPE: kind
     QA_RSPEC_TAGS: --tag health_check
+    QA_TESTS: ""
   before_script:
     - |
       redis_version=$(awk -F "=" "/MIN_REDIS_VERSION =/ {print \$2}" $CI_PROJECT_DIR/lib/system_check/app/redis_version_check.rb | sed "s/['\" ]//g")
       export EXTRA_DEPLOY_VALUES="--set redis.image.tag=${redis_version%.*}"
     - !reference [.cng-test, before_script]
+  rules:
+    - *mr-code-patterns # run validation for minimal supported redis version only on application code changes
 # ==========================================
 # Post test stage

qa/qa/specs/parallel_runner.rb

@@ -7,34 +7,40 @@ module Specs
     class ParallelRunner
       class << self
         def run(rspec_args)
-          used_processes = Runtime::Env.parallel_processes
-          args = [
+          cli_args = build_execution_args(rspec_args)
+          Runtime::Logger.debug("Using parallel runner to trigger tests with arguments: '#{cli_args}'")
+
+          set_environment!
+          perform_global_setup!
+
+          ParallelTests::CLI.new.run(cli_args)
+        end
+
+        private
+
+        delegate :parallel_processes, to: Runtime::Env
+
+        def build_execution_args(rspec_args)
+          specs = rspec_args.select { |arg| arg.include?("qa/specs/features") }
+          options = (rspec_args - specs).reject { |arg| arg == "--" }
+          # if amount of specs is less than parallel processes, use the amount of specs as count
+          # to avoid starting empty runs with no tests
+          used_processes = !specs.empty? && specs.size < parallel_processes ? specs.size : parallel_processes
+
+          cli_args = [
             "--type", "rspec",
             "-n", used_processes.to_s,
             "--serialize-stdout",
             '--first-is-1',
             "--combine-stderr"
           ]
-          unless rspec_args.include?('--')
-            index = rspec_args.index { |opt| opt.include?("qa/specs/features") }
-
-            rspec_args.insert(index, '--') if index
-          end
-
-          args.push("--", *rspec_args) unless rspec_args.empty?
-
-          Runtime::Logger.debug("Using parallel runner to trigger tests with arguments: '#{args}'")
-
-          set_environment!
-          perform_global_setup!
-
-          ParallelTests::CLI.new.run(args)
+          cli_args.push("--", *options) unless options.empty?
+          cli_args.push("--", *specs) unless specs.empty? # specific specs need to be seperated by additional "--"
+          cli_args
         end
-        private
-
         def perform_global_setup!
           Runtime::Browser.configure!
           Runtime::Release.perform_before_hooks

qa/qa/specs/spec_helper.rb

@@ -42,11 +42,7 @@
   config.example_status_persistence_file_path = ENV.fetch('RSPEC_LAST_RUN_RESULTS_FILE', 'tmp/examples.txt')
   config.prepend_before do |example|
-    if QA::Runtime::Env.parallel_run?
-      QA::Runtime::Logger.info("Starting test - PID #{Process.pid}: #{Rainbow(example.full_description).bright}")
-    else
-      QA::Runtime::Logger.info("Starting test: #{Rainbow(example.full_description).bright}")
-    end
+    QA::Runtime::Logger.info("Starting test: #{Rainbow(example.full_description).bright}")
     QA::Runtime::Example.current = example

qa/spec/specs/parallel_runner_spec.rb

@@ -8,45 +8,65 @@
   subject(:runner) { described_class }
   let(:parallel_tests) { instance_double(ParallelTests::CLI, run: nil) }
+  let(:parallel_processes) { 2 }
   before do
     allow(ParallelTests::CLI).to receive(:new).and_return(parallel_tests)
-    allow(Etc).to receive(:nprocessors).and_return(8)
+    allow(Etc).to receive(:nprocessors).and_return(parallel_processes)
     allow(ENV).to receive(:store)
     allow(QA::Runtime::Browser).to receive(:configure!)
     allow(QA::Runtime::Release).to receive(:perform_before_hooks)
     stub_env("QA_GITLAB_URL", "http://127.0.0.1:3000")
-    stub_env("QA_PARALLEL_PROCESSES", "8")
+    stub_env("QA_PARALLEL_PROCESSES", parallel_processes.to_s)
   end
-  it "runs cli without additional rspec args" do
-    runner.run([])
-
-    expect(parallel_tests).to have_received(:run).with([
+  def parallel_cli_args(processes = parallel_processes)
+    [
       "--type", "rspec",
-      "-n", "8",
+      "-n", processes.to_s,
       "--serialize-stdout",
       "--first-is-1",
       "--combine-stderr"
-    ])
+    ]
   end
-  it "runs cli with additional rspec args" do
-    runner.run(["--force-color", "qa/specs/features/api"])
-    expect(parallel_tests).to have_received(:run).with([
-      "--type", "rspec",
-      "-n", "8",
-      "--serialize-stdout",
-      "--first-is-1",
-      "--combine-stderr",
-      "--", "--force-color",
-      "--", "qa/specs/features/api"
-    ])
+  shared_examples "parallel cli runner" do |name, processes:, input_args:, received_args:|
+    it name do
+      runner.run(input_args)
+      expect(parallel_tests).to have_received(:run).with([*parallel_cli_args(processes), *received_args])
+    end
   end
+  it_behaves_like "parallel cli runner", "builds correct arguments without additional rspec args", {
+    processes: 2,
+    input_args: [],
+    received_args: []
+  }
+  it_behaves_like "parallel cli runner", "builds correct arguments with additional rspec args", {
+    processes: 2,
+    input_args: ['--force-color'],
+    received_args: ['--', '--force-color']
+  }
+  it_behaves_like "parallel cli runner", "builds correct arguments with specific specs", {
+    processes: 1,
+    input_args: ["qa/specs/features/api_spec.rb"],
+    received_args: ["--", "qa/specs/features/api_spec.rb"]
+  }
+  it_behaves_like "parallel cli runner", "builds correct arguments with specific specs and rspec options", {
+    processes: 2,
+    input_args: [
+      "--force-color",
+      "qa/specs/features/api_spec.rb", "qa/specs/features/api_2_spec.rb", "qa/specs/features/api_2_spec.rb"
+    ],
+    received_args: [
+      "--", "--force-color",
+      "--", "qa/specs/features/api_spec.rb", "qa/specs/features/api_2_spec.rb", "qa/specs/features/api_2_spec.rb"
+    ]
+  }
+
   context "with QA_GITLAB_URL not set" do
     before do
       stub_env("QA_GITLAB_URL", nil)
@@ -80,13 +100,7 @@
       actual_processes = QA::Runtime::Env.parallel_processes
       expect(parallel_tests).to have_received(:run) do |args|
-        expect(args).to eq([
-          "--type", "rspec",
-          "-n", actual_processes.to_s,
-          "--serialize-stdout",
-          "--first-is-1",
-          "--combine-stderr"
-        ])
+        expect(args).to eq(parallel_cli_args(actual_processes))
       end
     end
   end