Merge branch 'andrey-selective-execution-cng' into 'master'

Enable selective test execution for test-on-cng

Closes gitlab-org/quality/quality-engineering/team-tasks#3066

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169396



Merged-by: Nao Hashizume <nhashizume@gitlab.com>
Approved-by: Jay McCure <jmccure@gitlab.com>
Approved-by: Nao Hashizume <nhashizume@gitlab.com>
Reviewed-by: Jay McCure <jmccure@gitlab.com>
Co-authored-by: Andrejs Cunskis <acunskis@gitlab.com>

.gitlab/ci/test-on-cng/main.gitlab-ci.yml

@@ -10,6 +10,50 @@ workflow:
   rules:
     - when: always
+.mr-code-patterns: &mr-code-patterns
+  if: $MR_CODE_PATTERNS == "true"
+
+# list of specific tests is used when it is small enough to run without knapsack parallelization
+.tests-set: &tests-set
+  if: $QA_TESTS != ""
+
+# pattern is used in case when specific tests are selected but the amount of tests requires multiple jobs
+.parallel-pattern-set: &parallel-pattern-set
+  if: $KNAPSACK_TEST_FILE_PATTERN != ""
+
+.rules:selective:
+  rules:
+    # Skip selective jobs if mr contains app code patterns
+    - <<: *mr-code-patterns
+      when: never
+    - <<: *parallel-pattern-set
+      when: never
+    - <<: *tests-set
+
+.rules:selective-parallel:
+  rules:
+    # Skip selective jobs if mr contains app code patterns
+    - <<: *mr-code-patterns
+      when: never
+    - <<: *tests-set
+      when: never
+    - <<: *parallel-pattern-set
+
+.rules:full:
+  rules:
+    # Run full test suite for application code changes
+    # TODO: Remove once proper dynamic code mapping is in place - https://gitlab.com/groups/gitlab-org/quality/quality-engineering/-/epics/47
+    - if: $MR_CODE_PATTERNS == "true"
+      variables:
+        QA_TESTS: ""
+        KNAPSACK_TEST_FILE_PATTERN: ""
+    # Run full suite when both selective execution variables are unpopulated
+    - if: $QA_TESTS == "" && $KNAPSACK_TEST_FILE_PATTERN == ""
+    - <<: *tests-set
+      when: never
+    - <<: *parallel-pattern-set
+      when: never
+
 .cng-test:
   image: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:git-${GIT_VERSION}-lfs-${LFS_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-kubectl-${KUBECTL_VERSION}-helm-${HELM_VERSION}-kind-${KIND_VERSION}"
   stage: test
@@ -40,9 +84,6 @@ workflow:
     CNG_FORCE_COLOR: "true"
     QA_RUN_IN_PARALLEL: "true"
     QA_PARALLEL_PROCESSES: 4
-    # disable selective test execution until pipeline setup is implemented to support it correctly
-    KNAPSACK_TEST_FILE_PATTERN: ""
-    QA_TESTS: ""
   before_script:
     - echo "SUITE_RAN=true" > "$QA_SUITE_STATUS_ENV_FILE"
     # save extra values to be available for after_script if created dynamically
@@ -121,23 +162,51 @@ download-knapsack-report:
 # ==========================================
 # Test stage
 # ==========================================
+
+# == main 'instance' test suite ==
 cng-instance:
   extends: .cng-test-parallel
   variables:
     DEPLOYMENT_TYPE: kind
     EXTRA_DEPLOY_VALUES: --env COVERBAND_ENABLED=${COVERBAND_ENABLED}
-# Test run against environment with minimum supported redis version defined in lib/system_check/app/redis_version_check.rb
+  rules:
+    - if: $QA_SUITES !~ /Test::Instance::All/
+      when: never
+    - !reference [.rules:full, rules]
+cng-instance-selective-parallel:
+  extends: .cng-test-parallel
+  variables:
+    DEPLOYMENT_TYPE: kind
+  parallel: 2
+  rules:
+    - if: $QA_SUITES !~ /Test::Instance::All/
+      when: never
+    - !reference [.rules:selective-parallel, rules]
+
+cng-instance-selective:
+  extends: .cng-test
+  variables:
+    DEPLOYMENT_TYPE: kind
+  rules:
+    - if: $QA_SUITES !~ /Test::Instance::All/
+      when: never
+    - !reference [.rules:selective, rules]
+
+# == minimal supported redis version ==
 cng-qa-min-redis-version:
   extends: .cng-test
   variables:
     DEPLOYMENT_TYPE: kind
     QA_RSPEC_TAGS: --tag health_check
+    QA_TESTS: ""
   before_script:
     - |
       redis_version=$(awk -F "=" "/MIN_REDIS_VERSION =/ {print \$2}" $CI_PROJECT_DIR/lib/system_check/app/redis_version_check.rb | sed "s/['\" ]//g")
       export EXTRA_DEPLOY_VALUES="--set redis.image.tag=${redis_version%.*}"
     - !reference [.cng-test, before_script]
+  rules:
+    - *mr-code-patterns # run validation for minimal supported redis version only on application code changes
 # ==========================================
 # Post test stage

qa/qa/specs/parallel_runner.rb

@@ -7,34 +7,40 @@ module Specs
     class ParallelRunner
       class << self
         def run(rspec_args)
-          used_processes = Runtime::Env.parallel_processes
-          args = [
+          cli_args = build_execution_args(rspec_args)
+          Runtime::Logger.debug("Using parallel runner to trigger tests with arguments: '#{cli_args}'")
+
+          set_environment!
+          perform_global_setup!
+
+          ParallelTests::CLI.new.run(cli_args)
+        end
+
+        private
+
+        delegate :parallel_processes, to: Runtime::Env
+
+        def build_execution_args(rspec_args)
+          specs = rspec_args.select { |arg| arg.include?("qa/specs/features") }
+          options = (rspec_args - specs).reject { |arg| arg == "--" }
+          # if amount of specs is less than parallel processes, use the amount of specs as count
+          # to avoid starting empty runs with no tests
+          used_processes = !specs.empty? && specs.size < parallel_processes ? specs.size : parallel_processes
+
+          cli_args = [
             "--type", "rspec",
             "-n", used_processes.to_s,
             "--serialize-stdout",
             '--first-is-1',
             "--combine-stderr"
           ]
-          unless rspec_args.include?('--')
-            index = rspec_args.index { |opt| opt.include?("qa/specs/features") }
-
-            rspec_args.insert(index, '--') if index
-          end
-
-          args.push("--", *rspec_args) unless rspec_args.empty?
-
-          Runtime::Logger.debug("Using parallel runner to trigger tests with arguments: '#{args}'")
-
-          set_environment!
-          perform_global_setup!
-
-          ParallelTests::CLI.new.run(args)
+          cli_args.push("--", *options) unless options.empty?
+          cli_args.push("--", *specs) unless specs.empty? # specific specs need to be seperated by additional "--"
+          cli_args
         end
-        private
-
         def perform_global_setup!
           Runtime::Browser.configure!
           Runtime::Release.perform_before_hooks

qa/qa/specs/spec_helper.rb

@@ -42,11 +42,7 @@
   config.example_status_persistence_file_path = ENV.fetch('RSPEC_LAST_RUN_RESULTS_FILE', 'tmp/examples.txt')
   config.prepend_before do |example|
-    if QA::Runtime::Env.parallel_run?
-      QA::Runtime::Logger.info("Starting test - PID #{Process.pid}: #{Rainbow(example.full_description).bright}")
-    else
-      QA::Runtime::Logger.info("Starting test: #{Rainbow(example.full_description).bright}")
-    end
+    QA::Runtime::Logger.info("Starting test: #{Rainbow(example.full_description).bright}")
     QA::Runtime::Example.current = example

qa/spec/specs/parallel_runner_spec.rb

@@ -8,45 +8,65 @@
   subject(:runner) { described_class }
   let(:parallel_tests) { instance_double(ParallelTests::CLI, run: nil) }
+  let(:parallel_processes) { 2 }
   before do
     allow(ParallelTests::CLI).to receive(:new).and_return(parallel_tests)
-    allow(Etc).to receive(:nprocessors).and_return(8)
+    allow(Etc).to receive(:nprocessors).and_return(parallel_processes)
     allow(ENV).to receive(:store)
     allow(QA::Runtime::Browser).to receive(:configure!)
     allow(QA::Runtime::Release).to receive(:perform_before_hooks)
     stub_env("QA_GITLAB_URL", "http://127.0.0.1:3000")
-    stub_env("QA_PARALLEL_PROCESSES", "8")
+    stub_env("QA_PARALLEL_PROCESSES", parallel_processes.to_s)
   end
-  it "runs cli without additional rspec args" do
-    runner.run([])
-
-    expect(parallel_tests).to have_received(:run).with([
+  def parallel_cli_args(processes = parallel_processes)
+    [
       "--type", "rspec",
-      "-n", "8",
+      "-n", processes.to_s,
       "--serialize-stdout",
       "--first-is-1",
       "--combine-stderr"
-    ])
+    ]
   end
-  it "runs cli with additional rspec args" do
-    runner.run(["--force-color", "qa/specs/features/api"])
-    expect(parallel_tests).to have_received(:run).with([
-      "--type", "rspec",
-      "-n", "8",
-      "--serialize-stdout",
-      "--first-is-1",
-      "--combine-stderr",
-      "--", "--force-color",
-      "--", "qa/specs/features/api"
-    ])
+  shared_examples "parallel cli runner" do |name, processes:, input_args:, received_args:|
+    it name do
+      runner.run(input_args)
+      expect(parallel_tests).to have_received(:run).with([*parallel_cli_args(processes), *received_args])
+    end
   end
+  it_behaves_like "parallel cli runner", "builds correct arguments without additional rspec args", {
+    processes: 2,
+    input_args: [],
+    received_args: []
+  }
+  it_behaves_like "parallel cli runner", "builds correct arguments with additional rspec args", {
+    processes: 2,
+    input_args: ['--force-color'],
+    received_args: ['--', '--force-color']
+  }
+  it_behaves_like "parallel cli runner", "builds correct arguments with specific specs", {
+    processes: 1,
+    input_args: ["qa/specs/features/api_spec.rb"],
+    received_args: ["--", "qa/specs/features/api_spec.rb"]
+  }
+  it_behaves_like "parallel cli runner", "builds correct arguments with specific specs and rspec options", {
+    processes: 2,
+    input_args: [
+      "--force-color",
+      "qa/specs/features/api_spec.rb", "qa/specs/features/api_2_spec.rb", "qa/specs/features/api_2_spec.rb"
+    ],
+    received_args: [
+      "--", "--force-color",
+      "--", "qa/specs/features/api_spec.rb", "qa/specs/features/api_2_spec.rb", "qa/specs/features/api_2_spec.rb"
+    ]
+  }
+
   context "with QA_GITLAB_URL not set" do
     before do
       stub_env("QA_GITLAB_URL", nil)
@@ -80,13 +100,7 @@
       actual_processes = QA::Runtime::Env.parallel_processes
       expect(parallel_tests).to have_received(:run) do |args|
-        expect(args).to eq([
-          "--type", "rspec",
-          "-n", actual_processes.to_s,
-          "--serialize-stdout",
-          "--first-is-1",
-          "--combine-stderr"
-        ])
+        expect(args).to eq(parallel_cli_args(actual_processes))
       end
     end
   end