Merge branch 'qa-team-tasks-92-test-failure-retries' into 'master'

Retry failing tests

Closes gitlab-org/quality/team-tasks#92

See merge request gitlab-org/gitlab-ce!25391

(cherry picked from commit b570f53d)

d54cb37d Retry failed tests with rspec-retry

qa/Gemfile

@@ -8,3 +8,4 @@ gem 'rspec', '~> 3.7'
 gem 'selenium-webdriver', '~> 3.12'
 gem 'airborne', '~> 0.2.13'
 gem 'nokogiri', '~> 1.10.1'
+gem 'rspec-retry', '~> 0.6.1'

qa/Gemfile.lock

@@ -76,6 +76,8 @@ GEM
     rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
+    rspec-retry (0.6.1)
+      rspec-core (> 3.3)
     rspec-support (3.7.0)
     rubyzip (1.2.2)
     selenium-webdriver (3.141.0)
@@ -101,6 +103,7 @@ DEPENDENCIES
   pry-byebug (~> 3.5.1)
   rake (~> 12.3.0)
   rspec (~> 3.7)
+  rspec-retry (~> 0.6.1)
   selenium-webdriver (~> 3.12)
 BUNDLED WITH

qa/spec/spec_helper.rb

 require_relative '../qa'
+require 'rspec/retry'
 Dir[::File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f }
@@ -43,6 +44,17 @@ RSpec.configure do |config|
   config.profile_examples = 10
   config.order = :random
   Kernel.srand config.seed
+
+  # show retry status in spec process
+  config.verbose_retry = true
+
+  # show exception that triggers a retry if verbose_retry is set to true
+  config.display_try_failure_messages = true
+
+  config.around do |example|
+    retry_times = example.metadata.keys.include?(:quarantine) ? 1 : 3
+    example.run_with_retry retry: retry_times
+  end
 end
 # Skip tests in quarantine unless we explicitly focus on them.

qa/spec/spec_helper_spec.rb

@@ -28,6 +28,22 @@ describe 'rspec config tests' do
     end
   end
+  let(:group_2) do
+    RSpec.describe do
+      before(:all) do
+        @expectations = [1, 2, 3]
+      end
+
+      example 'not in quarantine' do
+        expect(@expectations.shift).to be(3)
+      end
+
+      example 'in quarantine', :quarantine do
+        expect(@expectations.shift).to be(3)
+      end
+    end
+  end
+
   context 'with no tags focussed' do
     before do
       group.run
@@ -301,4 +317,39 @@ describe 'rspec config tests' do
       end
     end
   end
+
+  context 'rspec retry' do
+    context 'in an untagged context' do
+      before do
+        group_2.run
+      end
+
+      it 'should run example :retry times' do
+        examples = group_2.descendant_filtered_examples
+        ex = examples.find { |e| e.description == 'not in quarantine' }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
+    end
+
+    context 'with :quarantine focussed' do
+      before do
+        RSpec.configure do |config|
+          config.inclusion_filter = :quarantine
+        end
+        group_2.run
+      end
+
+      after do
+        RSpec.configure do |config|
+          config.inclusion_filter.clear
+        end
+      end
+
+      it 'should run example once only' do
+        examples = group_2.descendant_filtered_examples
+        ex = examples.find { |e| e.description == 'in quarantine' }
+        expect(ex.execution_result.status).to eq(:failed)
+      end
+    end
+  end
 end