Skip to content
Snippets Groups Projects
Commit f7807a3c authored by Alexandru Croitor's avatar Alexandru Croitor
Browse files

Add auto-generation of iterations for iteration cadences

When an iteration cadence is set to automatic, we would generate a
number of future iteration automatically. This is based on running
a daily worker to generate needed iterations based on iteration
cadence settings.
parent 7a6e4f24
No related branches found
No related tags found
No related merge requests found
Showing
with 473 additions and 12 deletions
Loading
Loading
@@ -12,10 +12,11 @@ module HasUserType
ghost: 5,
project_bot: 6,
migration_bot: 7,
security_bot: 8
security_bot: 8,
automation_bot: 9
}.with_indifferent_access.freeze
 
BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot security_bot].freeze
BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot security_bot automation_bot].freeze
NON_INTERNAL_USER_TYPES = %w[human project_bot service_user].freeze
INTERNAL_USER_TYPES = (USER_TYPES.keys - NON_INTERNAL_USER_TYPES).freeze
 
Loading
Loading
Loading
Loading
@@ -794,6 +794,16 @@ def support_bot
end
end
 
def automation_bot
email_pattern = "automation%s@#{Settings.gitlab.host}"
unique_internal(where(user_type: :automation_bot), 'automation-bot', email_pattern) do |u|
u.bio = 'The GitLab automation bot used for automated workflows and tasks'
u.name = 'GitLab Automation Bot'
u.avatar = bot_avatar(image: 'support-bot.png') # todo: add an avatar for automation-bot
end
end
# Return true if there is only single non-internal user in the deployment,
# ghost user is ignored.
def single_user?
Loading
Loading
Loading
Loading
@@ -688,6 +688,9 @@
Settings.cron_jobs['iterations_update_status_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['iterations_update_status_worker']['cron'] ||= '5 0 * * *'
Settings.cron_jobs['iterations_update_status_worker']['job_class'] = 'IterationsUpdateStatusWorker'
Settings.cron_jobs['iterations_generator_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['iterations_generator_worker']['cron'] ||= '5 0 * * *'
Settings.cron_jobs['iterations_generator_worker']['job_class'] = 'Iterations::Cadences::ScheduleCreateIterationsWorker'
Settings.cron_jobs['vulnerability_statistics_schedule_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['vulnerability_statistics_schedule_worker']['cron'] ||= '15 1 * * *'
Settings.cron_jobs['vulnerability_statistics_schedule_worker']['job_class'] = 'Vulnerabilities::Statistics::ScheduleWorker'
Loading
Loading
Loading
Loading
@@ -3,6 +3,7 @@
module Iterations
class Cadence < ApplicationRecord
include Gitlab::SQL::Pattern
include EachBatch
 
self.table_name = 'iterations_cadences'
 
Loading
Loading
@@ -12,8 +13,8 @@ class Cadence < ApplicationRecord
validates :title, presence: true
validates :start_date, presence: true
validates :group_id, presence: true
validates :duration_in_weeks, presence: true
validates :iterations_in_advance, presence: true
validates :duration_in_weeks, presence: true, if: ->(cadence) { cadence.automatic? }
validates :iterations_in_advance, presence: true, if: ->(cadence) { cadence.automatic? }
validates :active, inclusion: [true, false]
validates :automatic, inclusion: [true, false]
validates :description, length: { maximum: 5000 }
Loading
Loading
@@ -23,9 +24,31 @@ class Cadence < ApplicationRecord
scope :is_automatic, -> (automatic) { where(automatic: automatic) }
scope :is_active, -> (active) { where(active: active) }
scope :ordered_by_title, -> { order(:title) }
scope :for_automated_iterations, -> do
table = Iterations::Cadence.arel_table
is_automatic(true)
.where(table[:duration_in_weeks].gt(0))
.where(
table[:last_run_date].eq(nil).or(
Arel::Nodes::NamedFunction.new('DATE',
[table[:last_run_date] + table[:duration_in_weeks] * Arel::Nodes::SqlLiteral.new("INTERVAL '1 week'")]
).lteq(Arel::Nodes::SqlLiteral.new('CURRENT_DATE'))
)
)
end
def next_open_iteration(date)
return unless date
iterations.without_state_enum(:closed).where('start_date >= ?', date).order(start_date: :asc).first
end
 
def self.search_title(query)
fuzzy_search(query, [:title])
end
def can_be_automated?
active? && automatic? && duration_in_weeks.to_i > 0 && iterations_in_advance.to_i > 0
end
end
end
# frozen_string_literal: true
module Iterations
module Cadences
class CreateIterationsInAdvanceService
def initialize(user, cadence)
@user = user
@cadence = cadence
end
def execute
return ::ServiceResponse.error(message: _('Operation not allowed'), http_status: 403) unless can_create_iterations_in_cadence?
return ::ServiceResponse.error(message: _('Cadence is not automated'), http_status: 422) unless cadence.can_be_automated?
iterations_batch = []
cadence_iterations = cadence.iterations
next_iteration_number = cadence_iterations.count + 1
last_iteration_due_date = cadence_iterations.order(:due_date).last&.due_date # rubocop: disable CodeReuse/ActiveRecord
next_start_date = last_iteration_due_date + 1.day if last_iteration_due_date
next_start_date ||= cadence.start_date
max_start_date = Date.today + (cadence.duration_in_weeks * cadence.iterations_in_advance).weeks
while next_start_date < max_start_date
iteration = build_iteration(cadence, next_start_date, next_iteration_number)
if iteration.valid?
iterations_batch << iteration
else
log_error(iteration)
end
next_iteration_number += 1
next_start_date = iteration.due_date + 1.day
end
new_iterations_collection = iterations_batch.map { |it| it.as_json(except: :id) }
::Gitlab::Database.bulk_insert(Iteration.table_name, new_iterations_collection) # rubocop:disable Gitlab/BulkInsert
cadence.update(last_run_date: Date.today)
::ServiceResponse.success
end
private
attr_accessor :user, :cadence
def log_error(iteration)
Gitlab::Services::Logger.error(
cadence: cadence.id,
group: cadence.group_id,
iteration_start_date: iteration.start_date,
iteration_due_date: iteration.due_date,
message: "Failed to generate iteration due to validation errors: #{iteration.errors.messages}"
)
end
def build_iteration(cadence, next_start_date, iteration_number)
current_time = Time.current
duration = cadence.duration_in_weeks
# because iteration start and due date are dates and not datetime and
# we do not allow for dates of 2 iterations to overlap a week ends-up being 6 days.
# i.e. instead of having smth like: 01/01/2020 00:00:00 - 01/08/2020 00:00:00
# we would convene to have 01/01/2020 00:00:00 - 01/07/2020 23:59:59 and because iteration dates have no time
# we end up having 01/01/2020(beginning of day) - 01/07/2020(end of day)
start_date = next_start_date
due_date = start_date + duration.weeks - 1.day
title = "Iteration #{iteration_number}: #{start_date.strftime(Date::DATE_FORMATS[:long])} - #{due_date.strftime(Date::DATE_FORMATS[:long])}"
description = "Auto-generated iteration for cadence##{cadence.id}: #{cadence.title} for period between #{start_date.strftime(Date::DATE_FORMATS[:long])} - #{due_date.strftime(Date::DATE_FORMATS[:long])}."
iteration = cadence.iterations.new(
created_at: current_time,
updated_at: current_time,
group: cadence.group,
start_date: start_date,
due_date: due_date,
title: title,
description: description
)
Iteration.with_group_iid_supply(cadence.group) do |supply|
iteration.iid = supply.next_value # this triggers an insert to internal_ids table
end
iteration
end
def can_create_iterations_in_cadence?
cadence && user && cadence.group.iteration_cadences_feature_flag_enabled? &&
(user.automation_bot? || user.can?(:create_iteration_cadence, cadence))
end
end
end
end
Loading
Loading
@@ -19,6 +19,7 @@ def execute
iteration_cadence = group.iterations_cadences.new(params)
 
if iteration_cadence.save
::Iterations::Cadences::CreateIterationsWorker.perform_async(iteration_cadence.id) if iteration_cadence.can_be_automated?
::ServiceResponse.success(payload: { iteration_cadence: iteration_cadence })
else
::ServiceResponse.error(message: iteration_cadence.errors.full_messages, http_status: 422)
Loading
Loading
Loading
Loading
@@ -323,6 +323,24 @@
:weight: 1
:idempotent:
:tags: []
- :name: cronjob:iterations_cadences_create_iterations
:worker_name: Iterations::Cadences::CreateIterationsWorker
:feature_category: :issue_tracking
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: cronjob:iterations_cadences_schedule_create_iterations
:worker_name: Iterations::Cadences::ScheduleCreateIterationsWorker
:feature_category: :issue_tracking
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: cronjob:iterations_update_status
:worker_name: IterationsUpdateStatusWorker
:feature_category: :issue_tracking
Loading
Loading
# frozen_string_literal: true
module Iterations
module Cadences
class CreateIterationsWorker
include ApplicationWorker
BATCH_SIZE = 1000
idempotent!
deduplicate :until_executed, including_scheduled: true
queue_namespace :cronjob
feature_category :issue_tracking
def perform(cadence_id)
cadence = ::Iterations::Cadence.find_by_id(cadence_id)
return unless cadence && cadence.group.iteration_cadences_feature_flag_enabled? # keep this behind FF for now
response = Iterations::Cadences::CreateIterationsInAdvanceService.new(automation_bot, cadence).execute
log_error(cadence, response) if response.error?
end
private
def log_error(cadence, response)
logger.error(
worker: self.class.name,
cadence_id: cadence&.id,
group_id: cadence&.group&.id,
message: response.message
)
end
def automation_bot
@automation_bot_id ||= User.automation_bot
end
end
end
end
# frozen_string_literal: true
module Iterations
module Cadences
class ScheduleCreateIterationsWorker
include ApplicationWorker
BATCH_SIZE = 1000
idempotent!
deduplicate :until_executed, including_scheduled: true
queue_namespace :cronjob
feature_category :issue_tracking
def perform
Iterations::Cadence.for_automated_iterations.each_batch(of: BATCH_SIZE) do |cadences|
cadences.each do |cadence|
Iterations::Cadences::CreateIterationsWorker.perform_async(cadence.id)
end
end
end
private
def log_error(cadence, response)
logger.error(
worker: self.class.name,
cadence_id: cadence&.id,
group_id: cadence&.group&.id,
message: response.message
)
end
def automation_bot
@automation_bot_id ||= User.automation_bot
end
end
end
end
Loading
Loading
@@ -7,8 +7,8 @@
 
factory :iterations_cadence, class: 'Iterations::Cadence' do
title
duration_in_weeks { 0 }
iterations_in_advance { 0 }
duration_in_weeks { 1 }
iterations_in_advance { 1 }
group
start_date { generate(:cadence_sequential_date) }
end
Loading
Loading
Loading
Loading
@@ -637,7 +637,7 @@
AND
("users"."user_type" IS NULL OR "users"."user_type" IN (NULL, 6, 4))
AND
("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 6, 1, 3, 7, 8))
("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 6, 1, 3, 7, 8, 9))
SQL
 
expect(users.to_sql.squish).to eq expected_sql.squish
Loading
Loading
@@ -665,7 +665,7 @@
AND
("users"."user_type" IS NULL OR "users"."user_type" IN (NULL, 6, 4))
AND
("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 6, 1, 3, 7, 8))
("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 6, 1, 3, 7, 8, 9))
AND
(EXISTS (SELECT 1 FROM "members"
WHERE "members"."user_id" = "users"."id"
Loading
Loading
Loading
Loading
@@ -90,7 +90,7 @@ def mutation_response
let(:attributes) { { title: '', duration_in_weeks: 1, active: false, automatic: false } }
 
it_behaves_like 'a mutation that returns errors in the response',
errors: ["Iterations in advance can't be blank", "Start date can't be blank", "Title can't be blank"]
errors: ["Start date can't be blank", "Title can't be blank"]
 
it 'does not create the iteration cadence' do
expect { post_graphql_mutation(mutation, current_user: current_user) }.not_to change(Iterations::Cadence, :count)
Loading
Loading
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Iterations::Cadences::CreateIterationsInAdvanceService do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:inactive_cadence) { create(:iterations_cadence, group: group, active: false, automatic: true, start_date: 2.weeks.ago) }
let_it_be(:manual_cadence) { create(:iterations_cadence, group: group, active: true, automatic: false, start_date: 2.weeks.ago) }
let_it_be_with_reload(:automated_cadence) { create(:iterations_cadence, group: group, active: true, automatic: true, start_date: 2.weeks.ago) }
subject { described_class.new(user, cadence).execute }
describe '#execute' do
context 'when user has permissions to create iterations' do
context 'when user is a group developer' do
before do
group.add_developer(user)
end
context 'with nil cadence' do
let(:cadence) { nil }
it 'returns error' do
expect(subject).to be_error
end
end
context 'with manual cadence' do
let(:cadence) { manual_cadence }
it 'returns error' do
expect(subject).to be_error
end
end
context 'with inactive cadence' do
let(:cadence) { inactive_cadence }
it 'returns error' do
expect(subject).to be_error
end
end
context 'with automatic and active cadence' do
let(:cadence) { automated_cadence }
it 'does not return error' do
expect(subject).not_to be_error
end
context 'when no iterations need to be created' do
let_it_be(:iteration) { create(:iteration, group: group, iterations_cadence: automated_cadence, start_date: 1.week.from_now, due_date: 2.weeks.from_now)}
it 'does not create any new iterations' do
expect { subject }.not_to change(Iteration, :count)
end
end
context 'when new iterations need to be created' do
context 'when no iterations exist' do
it 'creates new iterations' do
expect { subject }.to change(Iteration, :count).by(3)
end
end
context 'when advanced iterations exist but cadence needs to create more' do
let_it_be(:current_iteration) { create(:iteration, group: group, iterations_cadence: automated_cadence, start_date: 3.days.ago, due_date: (1.week - 3.days).from_now)}
let_it_be(:next_iteration) { create(:iteration, group: group, iterations_cadence: automated_cadence, start_date: current_iteration.due_date + 1.day, due_date: current_iteration.due_date + 1.week)}
before do
automated_cadence.iterations_in_advance = 2
end
it 'creates new iterations' do
expect { subject }.to change(Iteration, :count).by(1)
end
end
context 'when cadence has iterations but all are in the past' do
let_it_be(:past_iteration1) { create(:iteration, group: group, title: 'Iteration 1', iterations_cadence: automated_cadence, start_date: 3.weeks.ago, due_date: 2.weeks.ago)}
let_it_be(:past_iteration2) { create(:iteration, group: group, title: 'Iteration 2', iterations_cadence: automated_cadence, start_date: past_iteration1.due_date + 1.day, due_date: past_iteration1.due_date + 1.week)}
before do
automated_cadence.iterations_in_advance = 2
end
it 'creates new iterations' do
# because last iteration ended 1 week ago, we generate one iteration for current week and 2 in advance
expect { subject }.to change(Iteration, :count).by(3)
end
it 'updates cadence last_run_date' do
expect do
subject
end.to change(automated_cadence, :last_run_date).from(nil).to(Date.today)
end
it 'sets the titles correctly based on iterations count and follow-up dates' do
subject
initial_start_date = past_iteration2.due_date + 1.day
initial_due_date = past_iteration2.due_date + 1.week
expect(group.reload.iterations.pluck(:title)).to eq([
'Iteration 1',
'Iteration 2',
"Iteration 3: #{(initial_start_date).strftime(Date::DATE_FORMATS[:long])} - #{initial_due_date.strftime(Date::DATE_FORMATS[:long])}",
"Iteration 4: #{(initial_due_date + 1.day).strftime(Date::DATE_FORMATS[:long])} - #{(initial_due_date + 1.week).strftime(Date::DATE_FORMATS[:long])}",
"Iteration 5: #{(initial_due_date + 1.week + 1.day).strftime(Date::DATE_FORMATS[:long])} - #{(initial_due_date + 2.weeks).strftime(Date::DATE_FORMATS[:long])}"
])
end
end
end
end
end
end
end
end
Loading
Loading
@@ -94,6 +94,26 @@
expect { response }.to change { Iterations::Cadence.count }.by(1)
end
end
context 'when create cadence can be automated' do
it 'invokes worker to create iterations in advance' do
params[:automatic] = true
expect(::Iterations::Cadences::CreateIterationsWorker).to receive(:perform_async)
response
end
end
context 'when create cadence is not automated' do
it 'invokes worker to create iterations in advance' do
params[:automatic] = false
expect(::Iterations::Cadences::CreateIterationsWorker).not_to receive(:perform_async)
response
end
end
end
end
 
Loading
Loading
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Iterations::Cadences::CreateIterationsWorker do
let_it_be(:group) { create(:group) }
let_it_be(:start_date) { 3.weeks.ago }
let_it_be(:cadence) { create(:iterations_cadence, group: group, automatic: true, start_date: start_date, duration_in_weeks: 1, iterations_in_advance: 2) }
let(:mock_service) { double('mock_service', execute: ::ServiceResponse.success) }
subject(:worker) { described_class.new }
describe '#perform' do
context 'when passing in nil cadence id' do
it 'exits early' do
expect(Iterations::Cadences::CreateIterationsInAdvanceService).not_to receive(:new)
worker.perform(nil)
end
end
context 'when passing in non-existent cadence id' do
it 'exits early' do
expect(Iterations::Cadences::CreateIterationsInAdvanceService).not_to receive(:new)
worker.perform(non_existing_record_id)
end
end
context 'when passing existent cadence id' do
let(:mock_success_service) { double('mock_service', execute: ::ServiceResponse.success) }
let(:mock_error_service) { double('mock_service', execute: ::ServiceResponse.error(message: 'some error')) }
it 'invokes CreateIterationsInAdvanceService' do
expect(Iterations::Cadences::CreateIterationsInAdvanceService).to receive(:new).with(kind_of(User), kind_of(Iterations::Cadence)).and_return(mock_success_service)
expect(worker).not_to receive(:log_error)
worker.perform(cadence.id)
end
context 'when CreateIterationsInAdvanceService returns error' do
it 'logs error' do
allow(Iterations::Cadences::CreateIterationsInAdvanceService).to receive(:new).and_return(mock_error_service)
allow(mock_service).to receive(:execute)
expect(worker).to receive(:log_error)
worker.perform(cadence.id)
end
end
end
end
include_examples 'an idempotent worker' do
let(:job_args) { [cadence.id] }
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Iterations::Cadences::ScheduleCreateIterationsWorker do
let_it_be(:group) { create(:group) }
let_it_be(:start_date) { 3.weeks.ago }
let_it_be(:iteration_cadences) { create_list(:iterations_cadence, 2, group: group, automatic: true, start_date: start_date, duration_in_weeks: 1, iterations_in_advance: 2) }
subject(:worker) { described_class.new }
describe '#perform' do
context 'in batches' do
before do
stub_const("#{described_class}::BATCH_SIZE", 1)
end
it 'run in batches' do
expect(Iterations::Cadences::CreateIterationsWorker).to receive(:perform_async).twice
expect(Iterations::Cadence).to receive(:for_automated_iterations).and_call_original.once
worker.perform
end
end
end
include_examples 'an idempotent worker'
end
Loading
Loading
@@ -5782,6 +5782,9 @@ msgstr ""
msgid "CVE|Why Request a CVE ID?"
msgstr ""
 
msgid "Cadence is not automated"
msgstr ""
msgid "Callback URL"
msgstr ""
 
Loading
Loading
Loading
Loading
@@ -5,7 +5,7 @@
RSpec.describe User do
specify 'types consistency checks', :aggregate_failures do
expect(described_class::USER_TYPES.keys)
.to match_array(%w[human ghost alert_bot project_bot support_bot service_user security_bot visual_review_bot migration_bot])
.to match_array(%w[human ghost alert_bot project_bot support_bot service_user security_bot visual_review_bot migration_bot automation_bot])
expect(described_class::USER_TYPES).to include(*described_class::BOT_USER_TYPES)
expect(described_class::USER_TYPES).to include(*described_class::NON_INTERNAL_USER_TYPES)
expect(described_class::USER_TYPES).to include(*described_class::INTERNAL_USER_TYPES)
Loading
Loading
Loading
Loading
@@ -5274,9 +5274,10 @@ def access_levels(groups)
let_it_be(:user3) { create(:user, :ghost) }
let_it_be(:user4) { create(:user, user_type: :support_bot) }
let_it_be(:user5) { create(:user, state: 'blocked', user_type: :support_bot) }
let_it_be(:user6) { create(:user, user_type: :automation_bot) }
 
it 'returns all active users including active bots but ghost users' do
expect(described_class.active_without_ghosts).to match_array([user1, user4])
expect(described_class.active_without_ghosts).to match_array([user1, user4, user6])
end
end
 
Loading
Loading
@@ -5411,7 +5412,8 @@ def access_levels(groups)
{ user_type: :ghost },
{ user_type: :alert_bot },
{ user_type: :support_bot },
{ user_type: :security_bot }
{ user_type: :security_bot },
{ user_type: :automation_bot }
]
end
 
Loading
Loading
@@ -5467,6 +5469,7 @@ def access_levels(groups)
'alert_bot' | false
'support_bot' | false
'security_bot' | false
'automation_bot' | false
end
 
with_them do
Loading
Loading
@@ -5614,10 +5617,12 @@ def access_levels(groups)
it_behaves_like 'bot users', :migration_bot
it_behaves_like 'bot users', :security_bot
it_behaves_like 'bot users', :ghost
it_behaves_like 'bot users', :automation_bot
 
it_behaves_like 'bot user avatars', :alert_bot, 'alert-bot.png'
it_behaves_like 'bot user avatars', :support_bot, 'support-bot.png'
it_behaves_like 'bot user avatars', :security_bot, 'security-bot.png'
it_behaves_like 'bot user avatars', :automation_bot, 'support-bot.png'
 
context 'when bot is the support_bot' do
subject { described_class.support_bot }
Loading
Loading
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment