Commit 9cc3f365 authored by Fabio Pitino's avatar Fabio Pitino Committed by Kamil Trzciński
Browse files

Introduce a root level CI YAML

Add a layer of CI configuration on top of the .gitlab-ci.yml
which uses `include:` feature to include a file from repository,
from a remote URL or from a template (Auto-DevOps)
parent c4a8f041
......@@ -204,7 +204,7 @@ module Ci
end
 
scope :internal, -> { where(source: internal_sources) }
scope :ci_sources, -> { where(config_source: ci_sources_values) }
scope :ci_sources, -> { where(config_source: ::Ci::PipelineEnums.ci_config_sources_values) }
scope :for_user, -> (user) { where(user: user) }
scope :for_sha, -> (sha) { where(sha: sha) }
scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) }
......@@ -315,10 +315,6 @@ module Ci
sources.reject { |source| source == "external" }.values
end
 
def self.ci_sources_values
config_sources.values_at(:repository_source, :auto_devops_source, :unknown_source)
end
def self.bridgeable_statuses
::Ci::Pipeline::AVAILABLE_STATUSES - %w[created preparing pending]
end
......
......@@ -35,9 +35,20 @@ module Ci
{
unknown_source: nil,
repository_source: 1,
auto_devops_source: 2
auto_devops_source: 2,
remote_source: 4,
external_project_source: 5
}
end
def self.ci_config_sources_values
config_sources.values_at(
:unknown_source,
:repository_source,
:auto_devops_source,
:remote_source,
:external_project_source)
end
end
end
 
......
---
title: Allow CI config path to point to a URL or file in a different repository
merge_request: 20179
author:
type: added
......@@ -67,20 +67,37 @@ For information about setting a maximum artifact size for a project, see
 
## Custom CI configuration path
 
> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/12509) in GitLab 9.4.
> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/12509) in GitLab 9.4.
> - [Support for external `.gitlab-ci.yml` locations](https://gitlab.com/gitlab-org/gitlab/issues/14376) introduced in GitLab 12.6.
 
By default we look for the `.gitlab-ci.yml` file in the project's root
directory. If you require a different location **within** the repository,
you can set a custom path that will be used to look up the configuration file,
this path should be **relative** to the root.
directory. If needed, you can specify an alternate path and file name, including locations outside the project.
 
Here are some valid examples:
Hosting the configuration file in a separate project will allow stricter control of the
configuration file. You can limit access to the project hosting the configuration to only people
with proper authorization, and users can use the configuration for their pipelines,
without being able to modify it.
 
- `.gitlab-ci.yml`
If the CI configuration will stay within the repository, but in a
location different than the default,
the path must be relative to the root directory. Examples of valid paths and file names:
- `.gitlab-ci.yml` (default)
- `.my-custom-file.yml`
- `my/path/.gitlab-ci.yml`
- `my/path/.my-custom-file.yml`
 
If the CI configuration will be hosted in a different project within GitLab, the path must be relative
to the root directory in the other project, with the group and project name added to the end:
- `.gitlab-ci.yml@mygroup/another-project`
- `my/path/.my-custom-file.yml@mygroup/another-project`
If the CI configuration will be hosted on an external site, different than the GitLab instance,
the URL link must end with `.yml`:
- `http://example.com/generate/ci/config.yml`
The path can be customized at a project level. To customize the path:
 
1. Go to the project's **Settings > CI / CD**.
......
......@@ -22,6 +22,29 @@ describe Ci::Pipeline do
end
end
 
describe '.ci_sources' do
subject { described_class.ci_sources }
let(:all_config_sources) { described_class.config_sources }
before do
all_config_sources.each do |source, _value|
create(:ci_pipeline, config_source: source)
end
end
it 'contains pipelines having CI only config sources' do
expect(subject.map(&:config_source)).to contain_exactly(
'auto_devops_source',
'external_project_source',
'remote_source',
'repository_source',
'unknown_source'
)
expect(subject.size).to be < all_config_sources.size
end
end
describe '#with_vulnerabilities scope' do
let!(:pipeline_1) { create(:ci_pipeline, project: project) }
let!(:pipeline_2) { create(:ci_pipeline, project: project) }
......
......@@ -8,21 +8,28 @@ module Gitlab
class Content < Chain::Base
include Chain::Helpers
 
def perform!
return if @command.config_content
if content = content_from_repo
@command.config_content = content
@pipeline.config_source = :repository_source
# TODO: we should persist ci_config_path
# @pipeline.config_path = ci_config_path
elsif content = content_from_auto_devops
@command.config_content = content
@pipeline.config_source = :auto_devops_source
end
SOURCES = [
Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime,
Gitlab::Ci::Pipeline::Chain::Config::Content::Repository,
Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject,
Gitlab::Ci::Pipeline::Chain::Config::Content::Remote,
Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops
].freeze
LEGACY_SOURCES = [
Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime,
Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyRepository,
Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyAutoDevops
].freeze
 
unless @command.config_content
return error("Missing #{ci_config_path} file")
def perform!
if config = find_config
# TODO: we should persist config_content
# @pipeline.config_content = config.content
@command.config_content = config.content
@pipeline.config_source = config.source
else
error('Missing CI config file')
end
end
 
......@@ -32,24 +39,21 @@ module Gitlab
 
private
 
def content_from_repo
return unless project
return unless @pipeline.sha
return unless ci_config_path
def find_config
sources.each do |source|
config = source.new(@pipeline, @command)
return config if config.exists?
end
 
project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
rescue GRPC::NotFound, GRPC::Internal
nil
end
 
def content_from_auto_devops
return unless project&.auto_devops_enabled?
Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
end
def ci_config_path
project.ci_config_path.presence || '.gitlab-ci.yml'
def sources
if Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true)
SOURCES
else
LEGACY_SOURCES
end
end
end
end
......
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content
class AutoDevops < Source
def content
strong_memoize(:content) do
next unless project&.auto_devops_enabled?
template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
YAML.dump('include' => [{ 'template' => template.full_name }])
end
end
def source
:auto_devops_source
end
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content
class ExternalProject < Source
def content
strong_memoize(:content) do
next unless external_project_path?
path_file, path_project = ci_config_path.split('@', 2)
YAML.dump('include' => [{ 'project' => path_project, 'file' => path_file }])
end
end
def source
:external_project_source
end
private
# Example: path/to/.gitlab-ci.yml@another-group/another-project
def external_project_path?
ci_config_path =~ /\A.+(yml|yaml)@.+\z/
end
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content
class LegacyAutoDevops < Source
def content
strong_memoize(:content) do
next unless project&.auto_devops_enabled?
template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
template.content
end
end
def source
:auto_devops_source
end
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content
class LegacyRepository < Source
def content
strong_memoize(:content) do
next unless project
next unless @pipeline.sha
next unless ci_config_path
project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
rescue GRPC::NotFound, GRPC::Internal
nil
end
end
def source
:repository_source
end
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content
class Remote < Source
def content
strong_memoize(:content) do
next unless ci_config_path =~ URI.regexp(%w[http https])
YAML.dump('include' => [{ 'remote' => ci_config_path }])
end
end
def source
:remote_source
end
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content
class Repository < Source
def content
strong_memoize(:content) do
next unless file_in_repository?
YAML.dump('include' => [{ 'local' => ci_config_path }])
end
end
def source
:repository_source
end
private
def file_in_repository?
return unless project
return unless @pipeline.sha
project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present?
rescue GRPC::NotFound, GRPC::Internal
nil
end
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content
class Runtime < Source
def content
@command.config_content
end
def source
# The only case when this source is used is when the config content
# is passed in as parameter to Ci::CreatePipelineService.
# This would only occur with parent/child pipelines which is being
# implemented.
# TODO: change source to return :runtime_source
# https://gitlab.com/gitlab-org/gitlab/merge_requests/21041
nil
end
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Config
class Content
class Source
include Gitlab::Utils::StrongMemoize
DEFAULT_YAML_FILE = '.gitlab-ci.yml'
def initialize(pipeline, command)
@pipeline = pipeline
@command = command
end
def exists?
strong_memoize(:exists) do
content.present?
end
end
def content
raise NotImplementedError
end
def source
raise NotImplementedError
end
def project
@project ||= @pipeline.project
end
def ci_config_path
@ci_config_path ||= project.ci_config_path.presence || DEFAULT_YAML_FILE
end
end
end
end
end
end
end
end
......@@ -706,7 +706,7 @@ describe 'Pipelines', :js do
click_on 'Run Pipeline'
end
 
it { expect(page).to have_content('Missing .gitlab-ci.yml file') }
it { expect(page).to have_content('Missing CI config file') }
it 'creates a pipeline after first request failed and a valid gitlab-ci.yml file is available when trying again' do
click_button project.default_branch
 
......
......@@ -44,7 +44,7 @@ describe Gitlab::Chat::Command do
let(:pipeline) { command.create_pipeline }
 
before do
stub_repository_ci_yaml_file(sha: project.commit.id)
stub_ci_pipeline_yaml_file(gitlab_ci_yaml)
 
project.add_developer(chat_name.user)
end
......
......@@ -30,7 +30,7 @@ describe Gitlab::Ci::Pipeline::Chain::Build do
let(:step) { described_class.new(pipeline, command) }
 
before do
stub_repository_ci_yaml_file(sha: anything)
stub_ci_pipeline_yaml_file(gitlab_ci_yaml)
end
 
it 'never breaks the chain' do
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Pipeline::Chain::Config::Content do
let(:project) { create(:project, ci_config_path: ci_config_path) }
let(:pipeline) { build(:ci_pipeline, project: project) }
let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project) }
subject { described_class.new(pipeline, command) }
describe '#perform!' do
context 'when feature flag is disabled' do
before do
stub_feature_flags(ci_root_config_content: false)
end
context 'when config is defined in a custom path in the repository' do
let(:ci_config_path) { 'path/to/config.yml' }
before do
expect(project.repository)
.to receive(:gitlab_ci_yml_for)
.with(pipeline.sha, ci_config_path)
.and_return('the-content')
end
it 'returns the content of the YAML file' do
subject.perform!
expect(pipeline.config_source).to eq 'repository_source'
expect(command.config_content).to eq('the-content')
end
end
context 'when config is defined remotely' do
let(:ci_config_path) { 'http://example.com/path/to/ci/config.yml' }
it 'does not support URLs and default to AutoDevops' do
subject.perform!
expect(pipeline.config_source).to eq 'auto_devops_source'
template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
expect(command.config_content).to eq(template.content)
end
end
context 'when config is defined in a separate repository' do
let(:ci_config_path) { 'path/to/.gitlab-ci.yml@another-group/another-repo' }
it 'does not support YAML from external repository and default to AutoDevops' do
subject.perform!
expect(pipeline.config_source).to eq 'auto_devops_source'
template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
expect(command.config_content).to eq(template.content)
end
end
context 'when config is defined in the default .gitlab-ci.yml' do
let(:ci_config_path) { nil }
before do
expect(project.repository)
.to receive(:gitlab_ci_yml_for)
.with(pipeline.sha, '.gitlab-ci.yml')
.and_return('the-content')
end
it 'returns the content of the canonical config file' do
subject.perform!
expect(pipeline.config_source).to eq 'repository_source'
expect(command.config_content).to eq('the-content')
end
end
context 'when config is the Auto-Devops template' do
let(:ci_config_path) { nil }
before do
expect(project).to receive(:auto_devops_enabled?).and_return(true)
end
it 'returns the content of AutoDevops template' do
subject.perform!
expect(pipeline.config_source).to eq 'auto_devops_source'
template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
expect(command.config_content).to eq(template.content)
end
end