Introduce Auto Rollback facility

This commit introduces the auto rollback facility.

app/models/deployment.rb

@@ -41,8 +41,8 @@ class Deployment < ApplicationRecord
   scope :visible, -> { where(status: %i[running success failed canceled]) }
   scope :stoppable, -> { where.not(on_stop: nil).where.not(deployable_id: nil).success }
   scope :active, -> { where(status: %i[created running]) }
-  scope :older_than, -> (deployment) { where('id < ?', deployment.id) }
-  scope :with_deployable, -> { includes(:deployable).where('deployable_id IS NOT NULL') }
+  scope :older_than, -> (deployment) { where('deployments.id < ?', deployment.id) }
+  scope :with_deployable, -> { joins('INNER JOIN ci_builds ON ci_builds.id = deployments.deployable_id').preload(:deployable) }
   FINISHED_STATUSES = %i[success failed canceled].freeze
@@ -149,6 +149,10 @@ def finalize_fast_destroy(params)
         project.repository.delete_refs(*ref_paths.flatten)
       end
     end
+
+    def latest_for_sha(sha)
+      where(sha: sha).order(id: :desc).take
+    end
   end
   def commit

app/models/environment.rb

@@ -60,6 +60,7 @@ class Environment < ApplicationRecord
             addressable_url: true
   delegate :stop_action, :manual_actions, to: :last_deployment, allow_nil: true
+  delegate :auto_rollback_enabled?, to: :project
   scope :available, -> { with_state(:available) }
   scope :stopped, -> { with_state(:stopped) }
@@ -240,10 +241,6 @@ def stop_action_available?
   def cancel_deployment_jobs!
     jobs = active_deployments.with_deployable
     jobs.each do |deployment|
-      # guard against data integrity issues,
-      # for example https://gitlab.com/gitlab-org/gitlab/-/issues/218659#note_348823660
-      next unless deployment.deployable
-
       Gitlab::OptimisticLocking.retry_lock(deployment.deployable) do |deployable|
         deployable.cancel! if deployable&.cancelable?
       end

changelogs/unreleased/introduce-auto-rollback-service.yml

+---
+title: Add database index for deployment rollback targets
+merge_request: 47159
+author:
+type: performance

db/post_migrate/20201112145311_add_index_on_sha_for_initial_deployments.rb

+# frozen_string_literal: true
+
+class AddIndexOnShaForInitialDeployments < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  NEW_INDEX_NAME = 'index_deployments_on_environment_status_sha'
+  OLD_INDEX_NAME = 'index_deployments_on_environment_id_and_status'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :deployments, %i[environment_id status sha], name: NEW_INDEX_NAME
+    remove_concurrent_index_by_name :deployments, OLD_INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :deployments, %i[environment_id status], name: OLD_INDEX_NAME
+    remove_concurrent_index_by_name :services, NEW_INDEX_NAME
+  end
+end

db/schema_migrations/20201112145311

+085bb21bdbe3d062b3000d63c111aab5ba75c7e049c32779cccac5c320583759
\ No newline at end of file

db/structure.sql

@@ -20697,7 +20697,7 @@ CREATE INDEX index_deployments_on_environment_id_and_id ON deployments USING btr
 CREATE INDEX index_deployments_on_environment_id_and_iid_and_project_id ON deployments USING btree (environment_id, iid, project_id);
-CREATE INDEX index_deployments_on_environment_id_and_status ON deployments USING btree (environment_id, status);
+CREATE INDEX index_deployments_on_environment_status_sha ON deployments USING btree (environment_id, status, sha);
 CREATE INDEX index_deployments_on_id_and_status_and_created_at ON deployments USING btree (id, status, created_at);

ee/app/services/deployments/auto_rollback_service.rb

+# frozen_string_literal: true
+
+module Deployments
+  class AutoRollbackService < ::BaseService
+    def execute(environment)
+      result = validate(environment)
+      return result unless result[:status] == :success
+
+      deployment = find_rollback_target(environment)
+      return error('Failed to find a rollback target.') unless deployment
+
+      new_deployment = rollback_to(deployment)
+      success(deployment: new_deployment)
+    end
+
+    private
+
+    def validate(environment)
+      unless environment.auto_rollback_enabled?
+        return error('Auto Rollback is not enabled on the project.')
+      end
+
+      if environment.has_running_deployments?
+        return error('There are running deployments on the environment.')
+      end
+
+      if ::Gitlab::ApplicationRateLimiter.throttled?(:auto_rollback_deployment, scope: [environment])
+        return error('Auto Rollback was recentlly trigged for the environment. It will be re-activated after a minute.')
+      end
+
+      success
+    end
+
+    def find_rollback_target(environment)
+      current_deployment = environment.last_deployment
+      return unless current_deployment
+
+      previous_commit_ids = current_deployment.commit&.parent_ids
+      return unless previous_commit_ids
+
+      rollback_target = environment.successful_deployments
+        .with_deployable
+        .latest_for_sha(previous_commit_ids)
+
+      return unless rollback_target && rollback_target.deployable.retryable?
+
+      rollback_target
+    end
+
+    def rollback_to(deployment)
+      Ci::Build.retry(deployment.deployable, deployment.deployed_by).deployment
+    end
+  end
+end

ee/app/workers/all_queues.yml

@@ -299,6 +299,14 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: deployment:deployments_auto_rollback
+  :feature_category: :continuous_delivery
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 3
+  :idempotent: true
+  :tags: []
 - :name: epics:epics_update_epics_dates
   :feature_category: :epics
   :has_external_dependencies:

ee/app/workers/deployments/auto_rollback_worker.rb

+# frozen_string_literal: true
+
+module Deployments
+  class AutoRollbackWorker
+    include ApplicationWorker
+
+    idempotent!
+    feature_category :continuous_delivery
+    queue_namespace :deployment
+
+    def perform(environment_id)
+      Environment.find_by_id(environment_id).try do |environment|
+        Deployments::AutoRollbackService.new(environment.project, nil)
+          .execute(environment)
+      end
+    end
+  end
+end

ee/spec/services/deployments/auto_rollback_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Deployments::AutoRollbackService, :clean_gitlab_redis_cache do
+  let_it_be(:maintainer) { create(:user) }
+  let_it_be(:project, refind: true) { create(:project, :repository) }
+  let_it_be(:environment, refind: true) { create(:environment, project: project) }
+  let_it_be(:commits) { project.repository.commits('master', limit: 2) }
+
+  let(:service) { described_class.new(project, nil) }
+
+  before_all do
+    project.add_maintainer(maintainer)
+    project.update!(auto_rollback_enabled: true)
+  end
+
+  shared_examples_for 'rollback failure' do
+    it 'returns an error' do
+      expect(subject[:status]).to eq(:error)
+      expect(subject[:message]).to eq(message)
+    end
+  end
+
+  describe '#execute' do
+    subject { service.execute(environment) }
+
+    before do
+      stub_licensed_features(auto_rollback: true)
+      commits.reverse_each { |commit| create_deployment(commit.id) }
+    end
+
+    it 'successfully roll back a deployment' do
+      expect { subject }.to change { Deployment.count }.by(1)
+
+      expect(subject[:status]).to eq(:success)
+      expect(subject[:deployment].sha).to eq(commits[1].id)
+    end
+
+    context 'when auto_rollback checkbox is disabled on the project' do
+      before do
+        environment.project.auto_rollback_enabled = false
+      end
+
+      it_behaves_like 'rollback failure' do
+        let(:message) { 'Auto Rollback is not enabled on the project.' }
+      end
+    end
+
+    context 'when project does not have an sufficient license' do
+      before do
+        stub_licensed_features(auto_rollback: false)
+      end
+
+      it_behaves_like 'rollback failure' do
+        let(:message) { 'Auto Rollback is not enabled on the project.' }
+      end
+    end
+
+    context 'when there are running deployments ' do
+      before do
+        create(:deployment, :running, environment: environment)
+      end
+
+      it_behaves_like 'rollback failure' do
+        let(:message) { 'There are running deployments on the environment.' }
+      end
+    end
+
+    context 'when auto rollback was triggered recently' do
+      before do
+        allow(::Gitlab::ApplicationRateLimiter).to receive(:throttled?) { true }
+      end
+
+      it_behaves_like 'rollback failure' do
+        let(:message) { 'Auto Rollback was recentlly trigged for the environment. It will be re-activated after a minute.' }
+      end
+    end
+
+    context 'when there are no deployments on the environment' do
+      before do
+        environment.deployments.fast_destroy_all
+      end
+
+      it_behaves_like 'rollback failure' do
+        let(:message) { 'Failed to find a rollback target.' }
+      end
+    end
+
+    context 'when there are no deployed commits in the repository' do
+      before do
+        environment.last_deployment.update!(sha: 'not-exist')
+      end
+
+      it_behaves_like 'rollback failure' do
+        let(:message) { 'Failed to find a rollback target.' }
+      end
+    end
+
+    context "when rollback target's deployable is not retryable" do
+      before do
+        environment.all_deployments.first.deployable.degenerate!
+      end
+
+      it_behaves_like 'rollback failure' do
+        let(:message) { 'Failed to find a rollback target.' }
+      end
+    end
+
+    context "when the user who performed deployments is no longer a project member" do
+      let(:external_user) { create(:user) }
+
+      before do
+        environment.all_deployments.first.deployable.update!(user: external_user)
+      end
+
+      it 'raises an error' do
+        expect { subject }.to raise_error(Gitlab::Access::AccessDeniedError)
+      end
+    end
+
+    def create_deployment(commit_id)
+      attributes = { project: project, ref: 'master', user: maintainer }
+      pipeline = create(:ci_pipeline, :success, sha: commit_id, **attributes)
+      build = create(:ci_build, :success, pipeline: pipeline, environment: environment.name, **attributes)
+      create(:deployment, :success, environment: environment, deployable: build, sha: commit_id, **attributes)
+    end
+  end
+end

ee/spec/workers/deployments/auto_rollback_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Deployments::AutoRollbackWorker do
+  let_it_be(:environment) { create(:environment) }
+  let(:worker) { described_class.new }
+
+  describe '#perform' do
+    subject { worker.perform(environment_id) }
+
+    let(:environment_id) { environment.id }
+
+    it 'executes the rollback service' do
+      expect_next_instance_of(Deployments::AutoRollbackService, environment.project, nil) do |service|
+        expect(service).to receive(:execute).with(environment)
+      end
+
+      subject
+    end
+
+    context 'when an environment does not exist' do
+      let(:environment_id) { non_existing_record_id }
+
+      it 'does not execute the rollback service' do
+        expect(Deployments::AutoRollbackService).not_to receive(:new)
+
+        subject
+      end
+    end
+  end
+end

lib/gitlab/application_rate_limiter.rb

@@ -34,7 +34,8 @@ def rate_limits
           group_testing_hook:           { threshold: 5, interval: 1.minute },
           profile_add_new_email:        { threshold: 5, interval: 1.minute },
           profile_resend_email_confirmation:  { threshold: 5, interval: 1.minute },
-          update_environment_canary_ingress:  { threshold: 1, interval: 1.minute }
+          update_environment_canary_ingress:  { threshold: 1, interval: 1.minute },
+          auto_rollback_deployment:           { threshold: 1, interval: 3.minutes }
         }.freeze
       end

spec/models/deployment_spec.rb

@@ -372,6 +372,7 @@
       it 'retrieves deployments with deployable builds' do
         with_deployable = create(:deployment)
         create(:deployment, deployable: nil)
+        create(:deployment, deployable_type: 'CommitStatus', deployable_id: non_existing_record_id)
         is_expected.to contain_exactly(with_deployable)
       end
@@ -392,6 +393,35 @@
     end
   end
+  describe 'latest_for_sha' do
+    subject { described_class.latest_for_sha(sha) }
+
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:commits) { project.repository.commits('master', limit: 2) }
+    let_it_be(:deployments) { commits.reverse.map { |commit| create(:deployment, project: project, sha: commit.id) } }
+    let(:sha) { commits.map(&:id) }
+
+    it 'finds the latest deployment with sha' do
+      is_expected.to eq(deployments.last)
+    end
+
+    context 'when sha is old' do
+      let(:sha) { commits.last.id }
+
+      it 'finds the latest deployment with sha' do
+        is_expected.to eq(deployments.first)
+      end
+    end
+
+    context 'when sha is nil' do
+      let(:sha) { nil }
+
+      it 'returns nothing' do
+        is_expected.to be_nil
+      end
+    end
+  end
+
   describe '#includes_commit?' do
     let(:project) { create(:project, :repository) }
     let(:environment) { create(:environment, project: project) }

spec/models/environment_spec.rb

@@ -1394,4 +1394,31 @@
       it { is_expected.to be(false) }
     end
   end
+
+  describe '#cancel_deployment_jobs!' do
+    subject { environment.cancel_deployment_jobs! }
+
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:environment, reload: true) { create(:environment, project: project) }
+    let!(:deployment) { create(:deployment, project: project, environment: environment, deployable: build) }
+    let!(:build) { create(:ci_build, :running, project: project, environment: environment) }
+
+    it 'cancels an active deployment job' do
+      subject
+
+      expect(build.reset).to be_canceled
+    end
+
+    context 'when deployable does not exist' do
+      before do
+        deployment.update_column(:deployable_id, non_existing_record_id)
+      end
+
+      it 'does not raise an error' do
+        expect { subject }.not_to raise_error
+
+        expect(build.reset).to be_running
+      end
+    end
+  end
 end