Commit 4cfd634a authored by Fabio Pitino's avatar Fabio Pitino Committed by Kamil Trzciński
Browse files

Extract Ci::Config model class

Move all config related methods from Ci::Pipeline
out into a dedicated class.
parent 5ecbde49
......@@ -898,12 +898,6 @@ module Ci
value.with_indifferent_access
end
end
def build_attributes_from_config
return {} unless pipeline.config_processor
pipeline.config_processor.build_attributes(name)
end
end
end
 
......
......@@ -551,23 +551,6 @@ module Ci
end
end
 
def stage_seeds
return [] unless config_processor
strong_memoize(:stage_seeds) do
seeds = config_processor.stages_attributes.inject([]) do |previous_stages, attributes|
seed = Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes, previous_stages)
previous_stages + [seed]
end
seeds.select(&:included?)
end
end
def seeds_size
stage_seeds.sum(&:size)
end
def has_kubernetes_active?
project.deployment_platform&.active?
end
......@@ -587,62 +570,14 @@ module Ci
end
end
 
def set_config_source
if ci_yaml_from_repo
self.config_source = :repository_source
elsif implied_ci_yaml_file
self.config_source = :auto_devops_source
end
end
##
# TODO, setting yaml_errors should be moved to the pipeline creation chain.
#
def config_processor
return unless ci_yaml_file
return @config_processor if defined?(@config_processor)
@config_processor ||= begin
::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha, user: user })
rescue Gitlab::Ci::YamlProcessor::ValidationError => e
self.yaml_errors = e.message
nil
rescue => ex
self.yaml_errors = "Undefined error (#{Labkit::Correlation::CorrelationId.current_id})"
Gitlab::Sentry.track_acceptable_exception(ex, extra: {
project_id: project.id,
sha: sha,
ci_yaml_file: ci_yaml_file_path
})
nil
end
end
def ci_yaml_file_path
# TODO: this logic is duplicate with Pipeline::Chain::Config::Content
# we should persist this is `ci_pipelines.config_path`
def config_path
return unless repository_source? || unknown_source?
 
project.ci_config_path.presence || '.gitlab-ci.yml'
end
 
def ci_yaml_file
return @ci_yaml_file if defined?(@ci_yaml_file)
@ci_yaml_file =
if auto_devops_source?
implied_ci_yaml_file
else
ci_yaml_from_repo
end
if @ci_yaml_file
@ci_yaml_file
else
self.yaml_errors = "Failed to load CI/CD config file for #{sha}"
nil
end
end
def has_yaml_errors?
yaml_errors.present?
end
......@@ -711,7 +646,7 @@ module Ci
def predefined_variables
Gitlab::Ci::Variables::Collection.new.tap do |variables|
variables.append(key: 'CI_PIPELINE_IID', value: iid.to_s)
variables.append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path)
variables.append(key: 'CI_CONFIG_PATH', value: config_path)
variables.append(key: 'CI_PIPELINE_SOURCE', value: source.to_s)
variables.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s)
variables.append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s)
......@@ -906,24 +841,6 @@ module Ci
 
private
 
def ci_yaml_from_repo
return unless project
return unless sha
return unless ci_yaml_file_path
project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
rescue GRPC::NotFound, GRPC::Internal
nil
end
def implied_ci_yaml_file
return unless project
if project.auto_devops_enabled?
Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
end
end
def pipeline_data
Gitlab::DataBuilder::Pipeline.build(self)
end
......
......@@ -7,12 +7,14 @@ module Ci
CreateError = Class.new(StandardError)
 
SEQUENCE = [Gitlab::Ci::Pipeline::Chain::Build,
Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs,
Gitlab::Ci::Pipeline::Chain::Validate::Abilities,
Gitlab::Ci::Pipeline::Chain::Validate::Repository,
Gitlab::Ci::Pipeline::Chain::Validate::Config,
Gitlab::Ci::Pipeline::Chain::Config::Content,
Gitlab::Ci::Pipeline::Chain::Config::Process,
Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs,
Gitlab::Ci::Pipeline::Chain::Skip,
Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules,
Gitlab::Ci::Pipeline::Chain::Seed,
Gitlab::Ci::Pipeline::Chain::Limit::Size,
Gitlab::Ci::Pipeline::Chain::Populate,
Gitlab::Ci::Pipeline::Chain::Create,
......
......@@ -39,10 +39,6 @@
%th
= render partial: "projects/stage/stage", collection: pipeline.legacy_stages, as: :stage
 
- elsif pipeline.project.builds_enabled? && !pipeline.ci_yaml_file
.bs-callout.bs-callout-warning
= _("%{gitlab_ci_yml} not found in this commit") % { gitlab_ci_yml: ".gitlab-ci.yml" }
- if @pipeline.failed_builds.present?
#js-tab-failures.build-failures.tab-pane.build-page
%table.table.responsive-table.ci-table.responsive-table-sm-rounded
......
......@@ -17,7 +17,7 @@ module EE
super
 
@limit = Pipeline::Quota::Size
.new(project.namespace, pipeline)
.new(project.namespace, pipeline, command)
end
 
override :perform!
......
......@@ -9,9 +9,10 @@ module EE
include ::Gitlab::Utils::StrongMemoize
include ActionView::Helpers::TextHelper
 
def initialize(namespace, pipeline)
def initialize(namespace, pipeline, command)
@namespace = namespace
@pipeline = pipeline
@command = command
end
 
def enabled?
......@@ -34,7 +35,7 @@ module EE
private
 
def excessive_seeds_count
@excessive ||= @pipeline.seeds_size - ci_pipeline_size_limit
@excessive ||= seeds_size - ci_pipeline_size_limit
end
 
def ci_pipeline_size_limit
......@@ -42,6 +43,10 @@ module EE
@namespace.actual_limits.ci_pipeline_size
end
end
def seeds_size
@command.stage_seeds.sum(&:size) # rubocop: disable CodeReuse/ActiveRecord
end
end
end
end
......
......@@ -11,7 +11,12 @@ describe EE::Gitlab::Ci::Pipeline::Quota::Size do
 
let(:pipeline) { build_stubbed(:ci_pipeline, project: project) }
 
subject { described_class.new(namespace, pipeline) }
let(:command) do
double(:command,
stage_seeds: [double(:seed_1, size: 1), double(:seed_2, size: 1)])
end
subject { described_class.new(namespace, pipeline, command) }
 
shared_context 'pipeline size limit exceeded' do
before do
......
......@@ -10,8 +10,10 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
let(:pipeline) { build(:ci_pipeline, project: project) }
 
let(:command) do
double('command', project: project,
current_user: user)
double(:command,
project: project,
current_user: user,
stage_seeds: [double(:seed_1, size: 1), double(:seed_2, size: 1)])
end
 
let(:step) { described_class.new(pipeline, command) }
......@@ -31,9 +33,11 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
 
context 'when saving incomplete pipelines' do
let(:command) do
double('command', project: project,
current_user: user,
save_incompleted: true)
double(:command,
project: project,
current_user: user,
save_incompleted: true,
stage_seeds: [double(:seed_1, size: 1), double(:seed_2, size: 1)])
end
 
it 'drops the pipeline' do
......@@ -79,9 +83,11 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
 
context 'when not saving incomplete pipelines' do
let(:command) do
double('command', project: project,
current_user: user,
save_incompleted: false)
double(:command,
project: project,
current_user: user,
save_incompleted: false,
stage_seeds: [double(:seed_1, size: 1), double(:seed_2, size: 1)])
end
 
it 'does not drop the pipeline' do
......
......@@ -471,27 +471,6 @@ describe Ci::Pipeline do
end
end
 
describe '#ci_yaml_file_path' do
subject { pipeline.ci_yaml_file_path }
context 'the source is the repository' do
let(:implied_yml) { Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content }
before do
pipeline.repository_source!
end
it 'returns the configuration if found' do
allow(pipeline.project.repository).to receive(:gitlab_ci_yml_for)
.and_return('config')
expect(pipeline.ci_yaml_file).to be_a(String)
expect(pipeline.ci_yaml_file).not_to eq(implied_yml)
expect(pipeline.yaml_errors).to be_nil
end
end
end
describe '#latest_merge_request_pipeline?' do
subject { pipeline.latest_merge_request_pipeline? }
 
......
......@@ -64,7 +64,6 @@ describe Ci::CreatePipelineService, '#execute' do
 
expect(pipeline).to be_persisted
expect(pipeline).to be_failed
expect(pipeline.seeds_size).to be > 2
expect(pipeline.statuses).to be_empty
expect(pipeline.size_limit_exceeded?).to be true
end
......
......@@ -5,7 +5,7 @@ module Gitlab
module Pipeline
module Chain
class Base
attr_reader :pipeline, :command
attr_reader :pipeline, :command, :config
 
delegate :project, :current_user, to: :command
 
......
......@@ -22,8 +22,6 @@ module Gitlab
external_pull_request: @command.external_pull_request,
variables_attributes: Array(@command.variables_attributes)
)
@pipeline.set_config_source
end
 
def break?
......
......@@ -10,7 +10,9 @@ module Gitlab
:trigger_request, :schedule, :merge_request, :external_pull_request,
:ignore_skip_ci, :save_incompleted,
:seeds_block, :variables_attributes, :push_options,
:chat_data, :allow_mirror_update
:chat_data, :allow_mirror_update,
# These attributes are set by Chains during processing:
:config_content, :config_processor, :stage_seeds
) do
include Gitlab::Utils::StrongMemoize
 
......
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content < Chain::Base
include Chain::Helpers
def perform!
return if @command.config_content
if content = content_from_repo
@command.config_content = content
@pipeline.config_source = :repository_source
# TODO: we should persist ci_config_path
# @pipeline.config_path = ci_config_path
elsif content = content_from_auto_devops
@command.config_content = content
@pipeline.config_source = :auto_devops_source
end
unless @command.config_content
return error("Missing #{ci_config_path} file")
end
end
def break?
@pipeline.errors.any? || @pipeline.persisted?
end
private
def content_from_repo
return unless project
return unless @pipeline.sha
return unless ci_config_path
project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
rescue GRPC::NotFound, GRPC::Internal
nil
end
def content_from_auto_devops
return unless project&.auto_devops_enabled?
Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
end
def ci_config_path
project.ci_config_path.presence || '.gitlab-ci.yml'
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Process < Chain::Base
include Chain::Helpers
def perform!
raise ArgumentError, 'missing config content' unless @command.config_content
@command.config_processor = ::Gitlab::Ci::YamlProcessor.new(
@command.config_content, {
project: project,
sha: @pipeline.sha,
user: current_user
}
)
rescue Gitlab::Ci::YamlProcessor::ValidationError => ex
error(ex.message, config_error: true)
rescue => ex
Gitlab::Sentry.track_acceptable_exception(ex, extra: {
project_id: project.id,
sha: @pipeline.sha
})
error("Undefined error (#{Labkit::Correlation::CorrelationId.current_id})",
config_error: true)
end
def break?
@pipeline.errors.any? || @pipeline.persisted?
end
end
end
end
end
end
end
......@@ -41,7 +41,7 @@ module Gitlab
end
 
def workflow_config
@pipeline.config_processor.workflow_attributes || {}
@command.config_processor.workflow_attributes || {}
end
end
end
......
......@@ -10,29 +10,12 @@ module Gitlab
PopulateError = Class.new(StandardError)
 
def perform!
# Allocate next IID. This operation must be outside of transactions of pipeline creations.
pipeline.ensure_project_iid!
# Protect the pipeline. This is assigned in Populate instead of
# Build to prevent erroring out on ambiguous refs.
pipeline.protected = @command.protected_ref?
##
# Populate pipeline with block argument of CreatePipelineService#execute.
#
@command.seeds_block&.call(pipeline)
##
# Gather all runtime build/stage errors
#
if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence
return error(seeds_errors.join("\n"), config_error: true)
end
raise ArgumentError, 'missing stage seeds' unless @command.stage_seeds
 
##
# Populate pipeline with all stages, and stages with builds.
#
pipeline.stages = pipeline.stage_seeds.map(&:to_resource)
pipeline.stages = @command.stage_seeds.map(&:to_resource)
 
if pipeline.stages.none?
return error('No stages / jobs for this pipeline.')
......
......@@ -6,11 +6,13 @@ module Gitlab
module Chain
class RemoveUnwantedChatJobs < Chain::Base
def perform!
return unless pipeline.config_processor && pipeline.chat?
raise ArgumentError, 'missing config processor' unless @command.config_processor
return unless pipeline.chat?
 
# When scheduling a chat pipeline we only want to run the build
# that matches the chat command.
pipeline.config_processor.jobs.select! do |name, _|
@command.config_processor.jobs.select! do |name, _|
name.to_s == command.chat_data[:command].to_s
end
end
......
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
class Seed < Chain::Base
include Chain::Helpers
include Gitlab::Utils::StrongMemoize
def perform!
raise ArgumentError, 'missing config processor' unless @command.config_processor
# Allocate next IID. This operation must be outside of transactions of pipeline creations.
pipeline.ensure_project_iid!
# Protect the pipeline. This is assigned in Populate instead of
# Build to prevent erroring out on ambiguous refs.
pipeline.protected = @command.protected_ref?
##
# Populate pipeline with block argument of CreatePipelineService#execute.
#
@command.seeds_block&.call(pipeline)
##
# Gather all runtime build/stage errors
#
if stage_seeds_errors
return error(stage_seeds_errors.join("\n"), config_error: true)
end
@command.stage_seeds = stage_seeds
end
def break?
pipeline.errors.any?
end
private
def stage_seeds_errors
stage_seeds.flat_map(&:errors).compact.presence
end
def stage_seeds
strong_memoize(:stage_seeds) do
seeds = stages_attributes.inject([]) do |previous_stages, attributes|
seed = Gitlab::Ci::Pipeline::Seed::Stage.new(pipeline, attributes, previous_stages)
previous_stages + [seed]
end
seeds.select(&:included?)
end
end
def stages_attributes
@command.config_processor.stages_attributes
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Validate
class Config < Chain::Base
include Chain::Helpers
def perform!
unless @pipeline.config_processor
unless @pipeline.ci_yaml_file
return error("Missing #{@pipeline.ci_yaml_file_path} file")
end
if @command.save_incompleted && @pipeline.has_yaml_errors?
@pipeline.drop!(:config_error)
end
error(@pipeline.yaml_errors)
end
end
def break?
@pipeline.errors.any? || @pipeline.persisted?
end
end
end
end
end
end
end
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment