diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 432f3f242eb4e9cd30458b3207ed5165ed392436..416a2a333785b435b6bfad2d39f5f9024061f8dd 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -96,6 +96,14 @@ module Ci
BuildSuccessWorker.perform_async(id)
end
end
+
+ before_transition any => [:failed] do |build|
+ next if build.retries_max.zero?
+
+ if build.retries_count < build.retries_max
+ Ci::Build.retry(build, build.user)
+ end
+ end
end
def detailed_status(current_user)
@@ -130,6 +138,14 @@ module Ci
success? || failed? || canceled?
end
+ def retries_count
+ pipeline.builds.retried.where(name: self.name).count
+ end
+
+ def retries_max
+ self.options.fetch(:retry, 0).to_i
+ end
+
def latest?
!retried?
end
diff --git a/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml b/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml
new file mode 100644
index 0000000000000000000000000000000000000000..bdafc5929c02eaf743b21a595707695d6ae8694f
--- /dev/null
+++ b/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml
@@ -0,0 +1,4 @@
+---
+title: Allow to configure automatic retry of a failed CI/CD job
+merge_request: 12909
+author:
diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md
index 724843a4d565dd58c660d3fc26c7b80a72da577c..e12ef6e26851548ab12f181b30012266d438ffd3 100644
--- a/doc/ci/yaml/README.md
+++ b/doc/ci/yaml/README.md
@@ -395,6 +395,7 @@ job_name:
| after_script | no | Override a set of commands that are executed after job |
| environment | no | Defines a name of environment to which deployment is done by this job |
| coverage | no | Define code coverage settings for a given job |
+| retry | no | Define how many times a job can be auto-retried in case of a failure |
### script
@@ -1129,9 +1130,33 @@ A simple example:
```yaml
job1:
+ script: rspec
coverage: '/Code coverage: \d+\.\d+/'
```
+### retry
+
+**Notes:**
+- [Introduced][ce-3442] in GitLab 9.5.
+
+`retry` allows you to configure how many times a job is going to be retried in
+case of a failure.
+
+When a job fails, and has `retry` configured it is going to be processed again
+up to the amount of times specified by the `retry` keyword.
+
+If `retry` is set to 2, and a job succeeds in a second run (first retry), it won't be retried
+again. `retry` value has to be a positive integer, equal or larger than 0, but
+lower or equal to 2 (two retries maximum, three runs in total).
+
+A simple example:
+
+```yaml
+test:
+ script: rspec
+ retry: 2
+```
+
## Git Strategy
> Introduced in GitLab 8.9 as an experimental feature. May change or be removed
@@ -1506,3 +1531,4 @@ CI with various languages.
[variables]: ../variables/README.md
[ce-7983]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7983
[ce-7447]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7447
+[ce-3442]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3442
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index cf3a0336792a57f7f2a51a467efbfbfc13ad0d15..3a4911b23b0f955172e9885ab1ca2d75036b8dab 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -83,7 +83,8 @@ module Ci
before_script: job[:before_script],
script: job[:script],
after_script: job[:after_script],
- environment: job[:environment]
+ environment: job[:environment],
+ retry: job[:retry]
}.compact }
end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 176301bcca1bf98e1dedeecffad18ef2139de311..32f5c6ab1421606256bd409ab7afdf2813d0f3e9 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -11,7 +11,7 @@ module Gitlab
ALLOWED_KEYS = %i[tags script only except type image services allow_failure
type stage when artifacts cache dependencies before_script
- after_script variables environment coverage].freeze
+ after_script variables environment coverage retry].freeze
validations do
validates :config, allowed_keys: ALLOWED_KEYS
@@ -23,6 +23,9 @@ module Gitlab
with_options allow_nil: true do
validates :tags, array_of_strings: true
validates :allow_failure, boolean: true
+ validates :retry, numericality: { only_integer: true,
+ greater_than_or_equal_to: 0,
+ less_than_or_equal_to: 2 }
validates :when,
inclusion: { in: %w[on_success on_failure always manual],
message: 'should be on_success, on_failure, ' \
@@ -76,9 +79,9 @@ module Gitlab
helpers :before_script, :script, :stage, :type, :after_script,
:cache, :image, :services, :only, :except, :variables,
- :artifacts, :commands, :environment, :coverage
+ :artifacts, :commands, :environment, :coverage, :retry
- attributes :script, :tags, :allow_failure, :when, :dependencies
+ attributes :script, :tags, :allow_failure, :when, :dependencies, :retry
def compose!(deps = nil)
super do
@@ -142,6 +145,7 @@ module Gitlab
environment: environment_defined? ? environment_value : nil,
environment_name: environment_defined? ? environment_value[:name] : nil,
coverage: coverage_defined? ? coverage_value : nil,
+ retry: retry_defined? ? retry_value.to_i : nil,
artifacts: artifacts_value,
after_script: after_script_value,
ignore: ignored? }
diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb
index a77f01ecb0022382f6420cede7c3a81e17e03281..678cebe365b3df40c585e1949e20db4cf5a252f1 100644
--- a/spec/factories/ci/builds.rb
+++ b/spec/factories/ci/builds.rb
@@ -84,6 +84,10 @@ FactoryGirl.define do
success
end
+ trait :retried do
+ retried true
+ end
+
trait :cancelable do
pending
end
diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
index ea79389e67e8af7c1ca9bbd5baa3069fcaf4e6ee..ed571a2ba051b196d896a1aadf9f61a918b1d70f 100644
--- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
+++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
@@ -32,6 +32,28 @@ module Ci
end
end
+ describe 'retry entry' do
+ context 'when retry count is specified' do
+ let(:config) do
+ YAML.dump(rspec: { script: 'rspec', retry: 1 })
+ end
+
+ it 'includes retry count in build options attribute' do
+ expect(subject[:options]).to include(retry: 1)
+ end
+ end
+
+ context 'when retry count is not specified' do
+ let(:config) do
+ YAML.dump(rspec: { script: 'rspec' })
+ end
+
+ it 'does not persist retry count in the database' do
+ expect(subject[:options]).not_to have_key(:retry)
+ end
+ end
+ end
+
describe 'allow failure entry' do
context 'when job is a manual action' do
context 'when allow_failure is defined' do
diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb
index c5cad887b64c51031eb1f0cbe305c9d255b20894..6769f64f95055c6aadfc75bb4ddbd718d8d92d1c 100644
--- a/spec/lib/gitlab/ci/config/entry/job_spec.rb
+++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb
@@ -80,6 +80,45 @@ describe Gitlab::Ci::Config::Entry::Job do
expect(entry.errors).to include "job script can't be blank"
end
end
+
+ context 'when retry value is not correct' do
+ context 'when it is not a numeric value' do
+ let(:config) { { retry: true } }
+
+ it 'returns error about invalid type' do
+ expect(entry).not_to be_valid
+ expect(entry.errors).to include 'job retry is not a number'
+ end
+ end
+
+ context 'when it is lower than zero' do
+ let(:config) { { retry: -1 } }
+
+ it 'returns error about value too low' do
+ expect(entry).not_to be_valid
+ expect(entry.errors)
+ .to include 'job retry must be greater than or equal to 0'
+ end
+ end
+
+ context 'when it is not an integer' do
+ let(:config) { { retry: 1.5 } }
+
+ it 'returns error about wrong value' do
+ expect(entry).not_to be_valid
+ expect(entry.errors).to include 'job retry must be an integer'
+ end
+ end
+
+ context 'when the value is too high' do
+ let(:config) { { retry: 10 } }
+
+ it 'returns error about value too high' do
+ expect(entry).not_to be_valid
+ expect(entry.errors).to include 'job retry must be less than or equal to 2'
+ end
+ end
+ end
end
end
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 154b6759f46a33529ccdf5c9af8367140d61459a..0b521d720f3b8232532734945644dccf83db16ff 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -802,6 +802,47 @@ describe Ci::Build, :models do
end
end
+ describe 'build auto retry feature' do
+ describe '#retries_count' do
+ subject { create(:ci_build, name: 'test', pipeline: pipeline) }
+
+ context 'when build has been retried several times' do
+ before do
+ create(:ci_build, :retried, name: 'test', pipeline: pipeline)
+ create(:ci_build, :retried, name: 'test', pipeline: pipeline)
+ end
+
+ it 'reports a correct retry count value' do
+ expect(subject.retries_count).to eq 2
+ end
+ end
+
+ context 'when build has not been retried' do
+ it 'returns zero' do
+ expect(subject.retries_count).to eq 0
+ end
+ end
+ end
+
+ describe '#retries_max' do
+ context 'when max retries value is defined' do
+ subject { create(:ci_build, options: { retry: 1 }) }
+
+ it 'returns a number of configured max retries' do
+ expect(subject.retries_max).to eq 1
+ end
+ end
+
+ context 'when max retries value is not defined' do
+ subject { create(:ci_build) }
+
+ it 'returns zero' do
+ expect(subject.retries_max).to eq 0
+ end
+ end
+ end
+ end
+
describe '#keep_artifacts!' do
let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) }
@@ -1583,7 +1624,7 @@ describe Ci::Build, :models do
end
end
- describe 'State transition: any => [:pending]' do
+ describe 'state transition: any => [:pending]' do
let(:build) { create(:ci_build, :created) }
it 'queues BuildQueueWorker' do
@@ -1592,4 +1633,35 @@ describe Ci::Build, :models do
build.enqueue
end
end
+
+ describe 'state transition when build fails' do
+ context 'when build is configured to be retried' do
+ subject { create(:ci_build, :running, options: { retry: 3 }) }
+
+ it 'retries builds and assigns a same user to it' do
+ expect(described_class).to receive(:retry)
+ .with(subject, subject.user)
+
+ subject.drop!
+ end
+ end
+
+ context 'when build is not configured to be retried' do
+ subject { create(:ci_build, :running) }
+
+ it 'does not retry build' do
+ expect(described_class).not_to receive(:retry)
+
+ subject.drop!
+ end
+
+ it 'does not count retries when not necessary' do
+ expect(described_class).not_to receive(:retry)
+ expect_any_instance_of(described_class)
+ .not_to receive(:retries_count)
+
+ subject.drop!
+ end
+ end
+ end
end
diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb
index e71c462b99adaabe7155f026634790acba5f9842..ba07c01d43f1eff6d08d8b539f206031c0f96cb8 100644
--- a/spec/services/ci/create_pipeline_service_spec.rb
+++ b/spec/services/ci/create_pipeline_service_spec.rb
@@ -320,5 +320,19 @@ describe Ci::CreatePipelineService, :services do
end.not_to change { Environment.count }
end
end
+
+ context 'when builds with auto-retries are configured' do
+ before do
+ config = YAML.dump(rspec: { script: 'rspec', retry: 2 })
+ stub_ci_pipeline_yaml_file(config)
+ end
+
+ it 'correctly creates builds with auto-retry value configured' do
+ pipeline = execute_service
+
+ expect(pipeline).to be_persisted
+ expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 2
+ end
+ end
end
end
diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index efcaccc254e1e8a525e00e4ae596e550bb9bb8d9..0934833a4fa3561f3d7cd0f470da3ef3cc1da73e 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -463,6 +463,35 @@ describe Ci::ProcessPipelineService, '#execute', :services do
end
end
+ context 'when builds with auto-retries are configured' do
+ before do
+ create_build('build:1', stage_idx: 0, user: user, options: { retry: 2 })
+ create_build('test:1', stage_idx: 1, user: user, when: :on_failure)
+ create_build('test:2', stage_idx: 1, user: user, options: { retry: 1 })
+ end
+
+ it 'automatically retries builds in a valid order' do
+ expect(process_pipeline).to be_truthy
+
+ fail_running_or_pending
+
+ expect(builds_names).to eq %w[build:1 build:1]
+ expect(builds_statuses).to eq %w[failed pending]
+
+ succeed_running_or_pending
+
+ expect(builds_names).to eq %w[build:1 build:1 test:2]
+ expect(builds_statuses).to eq %w[failed success pending]
+
+ succeed_running_or_pending
+
+ expect(builds_names).to eq %w[build:1 build:1 test:2]
+ expect(builds_statuses).to eq %w[failed success success]
+
+ expect(pipeline.reload).to be_success
+ end
+ end
+
def process_pipeline
described_class.new(pipeline.project, user).execute(pipeline)
end