Merge branch 'backstage/gb/populating-pipeline-refactoring' into 'master'

Improve how we populate pipeline with stages and builds

Closes #43941

See merge request gitlab-org/gitlab-ce!17841

app/models/ci/pipeline.rb

@@ -6,6 +6,7 @@ module Ci
     include AfterCommitQueue
     include Presentable
     include Gitlab::OptimisticLocking
+    include Gitlab::Utils::StrongMemoize
     belongs_to :project, inverse_of: :pipelines
     belongs_to :user
@@ -361,21 +362,23 @@ module Ci
     def stage_seeds
       return [] unless config_processor
-      @stage_seeds ||= config_processor.stage_seeds(self)
+      strong_memoize(:stage_seeds) do
+        seeds = config_processor.stages_attributes.map do |attributes|
+          Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes)
+        end
+
+        seeds.select(&:included?)
+      end
     end
     def seeds_size
-      @seeds_size ||= stage_seeds.sum(&:size)
+      stage_seeds.sum(&:size)
     end
     def has_kubernetes_active?
       project.deployment_platform&.active?
     end
-    def has_stage_seeds?
-      stage_seeds.any?
-    end
-
     def has_warnings?
       builds.latest.failed_but_allowed.any?
     end
@@ -388,6 +391,9 @@ module Ci
       end
     end
+    ##
+    # TODO, setting yaml_errors should be moved to the pipeline creation chain.
+    #
     def config_processor
       return unless ci_yaml_file
       return @config_processor if defined?(@config_processor)
@@ -472,6 +478,14 @@ module Ci
       end
     end
+    def protected_ref?
+      strong_memoize(:protected_ref) { project.protected_for?(ref) }
+    end
+
+    def legacy_trigger
+      strong_memoize(:legacy_trigger) { trigger_requests.first }
+    end
+
     def predefined_variables
       Gitlab::Ci::Variables::Collection.new
         .append(key: 'CI_PIPELINE_ID', value: id.to_s)

app/services/ci/create_pipeline_service.rb

@@ -7,6 +7,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Validate::Repository,
                 Gitlab::Ci::Pipeline::Chain::Validate::Config,
                 Gitlab::Ci::Pipeline::Chain::Skip,
+                Gitlab::Ci::Pipeline::Chain::Populate,
                 Gitlab::Ci::Pipeline::Chain::Create].freeze
     def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, &block)

app/services/ci/create_pipeline_stages_service.rb

-module Ci
-  class CreatePipelineStagesService < BaseService
-    def execute(pipeline)
-      pipeline.stage_seeds.each do |seed|
-        seed.user = current_user
-
-        seed.create! do |build|
-          ##
-          # Create the environment before the build starts. This sets its slug and
-          # makes it available as an environment variable
-          #
-          if build.has_environment?
-            environment_name = build.expanded_environment_name
-            project.environments.find_or_create_by(name: environment_name)
-          end
-        end
-      end
-    end
-  end
-end

app/services/ci/pipeline_trigger_service.rb

@@ -16,8 +16,8 @@ module Ci
       pipeline = Ci::CreatePipelineService.new(project, trigger.owner, ref: params[:ref])
         .execute(:trigger, ignore_skip_ci: true) do |pipeline|
-          pipeline.trigger_requests.create!(trigger: trigger)
-          create_pipeline_variables!(pipeline)
+          pipeline.trigger_requests.build(trigger: trigger)
+          pipeline.variables.build(variables)
         end
       if pipeline.persisted?
@@ -33,14 +33,10 @@ module Ci
       end
     end
-    def create_pipeline_variables!(pipeline)
-      return unless params[:variables]
-
-      variables = params[:variables].map do |key, value|
+    def variables
+      params[:variables].to_h.map do |key, value|
         { key: key, value: value }
       end
-
-      pipeline.variables.create!(variables)
     end
   end
 end

lib/gitlab/ci/pipeline/chain/create.rb

@@ -9,11 +9,16 @@ module Gitlab
             ::Ci::Pipeline.transaction do
               pipeline.save!
-              @command.seeds_block&.call(pipeline)
-
-              ::Ci::CreatePipelineStagesService
-                .new(project, current_user)
-                .execute(pipeline)
+              ##
+              # Create environments before the pipeline starts.
+              #
+              pipeline.builds.each do |build|
+                if build.has_environment?
+                  project.environments.find_or_create_by(
+                    name: build.expanded_environment_name
+                  )
+                end
+              end
             end
           rescue ActiveRecord::RecordInvalid => e
             error("Failed to persist the pipeline: #{e}")

lib/gitlab/ci/pipeline/chain/populate.rb

+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Populate < Chain::Base
+          include Chain::Helpers
+
+          PopulateError = Class.new(StandardError)
+
+          def perform!
+            ##
+            # Populate pipeline with block argument of CreatePipelineService#execute.
+            #
+            @command.seeds_block&.call(pipeline)
+
+            ##
+            # Populate pipeline with all stages and builds from pipeline seeds.
+            #
+            pipeline.stage_seeds.each do |stage|
+              stage.user = current_user
+
+              pipeline.stages << stage.to_resource
+
+              stage.seeds.each do |build|
+                pipeline.builds << build.to_resource
+              end
+            end
+
+            if pipeline.stages.none?
+              return error('No stages / jobs for this pipeline.')
+            end
+
+            if pipeline.invalid?
+              return error('Failed to build the pipeline!')
+            end
+
+            raise Populate::PopulateError if pipeline.persisted?
+          end
+
+          def break?
+            pipeline.errors.any?
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/validate/config.rb

@@ -16,11 +16,7 @@ module Gitlab
                   @pipeline.drop!(:config_error)
                 end
-                return error(@pipeline.yaml_errors)
-              end
-
-              unless @pipeline.has_stage_seeds?
-                return error('No stages / jobs for this pipeline.')
+                error(@pipeline.yaml_errors)
               end
             end

lib/gitlab/ci/pipeline/seed/base.rb

+module Gitlab
+  module Ci
+    module Pipeline
+      module Seed
+        class Base
+          def attributes
+            raise NotImplementedError
+          end
+
+          def included?
+            raise NotImplementedError
+          end
+
+          def to_resource
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/seed/build.rb

+module Gitlab
+  module Ci
+    module Pipeline
+      module Seed
+        class Build < Seed::Base
+          include Gitlab::Utils::StrongMemoize
+
+          delegate :dig, to: :@attributes
+
+          def initialize(pipeline, attributes)
+            @pipeline = pipeline
+            @attributes = attributes
+
+            @only = attributes.delete(:only)
+            @except = attributes.delete(:except)
+          end
+
+          def user=(current_user)
+            @attributes.merge!(user: current_user)
+          end
+
+          def included?
+            strong_memoize(:inclusion) do
+              only_specs = Gitlab::Ci::Build::Policy.fabricate(@only)
+              except_specs = Gitlab::Ci::Build::Policy.fabricate(@except)
+
+              only_specs.all? { |spec| spec.satisfied_by?(@pipeline) } &&
+                except_specs.none? { |spec| spec.satisfied_by?(@pipeline) }
+            end
+          end
+
+          def attributes
+            @attributes.merge(
+              pipeline: @pipeline,
+              project: @pipeline.project,
+              ref: @pipeline.ref,
+              tag: @pipeline.tag,
+              trigger_request: @pipeline.legacy_trigger,
+              protected: @pipeline.protected_ref?
+            )
+          end
+
+          def to_resource
+            strong_memoize(:resource) do
+              ::Ci::Build.new(attributes)
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/seed/stage.rb

+module Gitlab
+  module Ci
+    module Pipeline
+      module Seed
+        class Stage < Seed::Base
+          include Gitlab::Utils::StrongMemoize
+
+          delegate :size, to: :seeds
+          delegate :dig, to: :seeds
+
+          def initialize(pipeline, attributes)
+            @pipeline = pipeline
+            @attributes = attributes
+
+            @builds = attributes.fetch(:builds).map do |attributes|
+              Seed::Build.new(@pipeline, attributes)
+            end
+          end
+
+          def user=(current_user)
+            @builds.each { |seed| seed.user = current_user }
+          end
+
+          def attributes
+            { name: @attributes.fetch(:name),
+              pipeline: @pipeline,
+              project: @pipeline.project }
+          end
+
+          def seeds
+            strong_memoize(:seeds) do
+              @builds.select(&:included?)
+            end
+          end
+
+          def included?
+            seeds.any?
+          end
+
+          def to_resource
+            strong_memoize(:stage) do
+              ::Ci::Stage.new(attributes).tap do |stage|
+                seeds.each { |seed| stage.builds << seed.to_resource }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/stage/seed.rb

-module Gitlab
-  module Ci
-    module Stage
-      class Seed
-        include ::Gitlab::Utils::StrongMemoize
-
-        attr_reader :pipeline
-
-        delegate :project, to: :pipeline
-        delegate :size, to: :@jobs
-
-        def initialize(pipeline, stage, jobs)
-          @pipeline = pipeline
-          @stage = { name: stage }
-          @jobs = jobs.to_a.dup
-        end
-
-        def user=(current_user)
-          @jobs.map! do |attributes|
-            attributes.merge(user: current_user)
-          end
-        end
-
-        def stage
-          @stage.merge(project: project)
-        end
-
-        def builds
-          trigger = pipeline.trigger_requests.first
-
-          @jobs.map do |attributes|
-            attributes.merge(project: project,
-                             ref: pipeline.ref,
-                             tag: pipeline.tag,
-                             trigger_request: trigger,
-                             protected: protected_ref?)
-          end
-        end
-
-        def create!
-          pipeline.stages.create!(stage).tap do |stage|
-            builds_attributes = builds.map do |attributes|
-              attributes.merge(stage_id: stage.id)
-            end
-
-            pipeline.builds.create!(builds_attributes).each do |build|
-              yield build if block_given?
-            end
-          end
-        end
-
-        private
-
-        def protected_ref?
-          strong_memoize(:protected_ref) do
-            project.protected_for?(pipeline.ref)
-          end
-        end
-      end
-    end
-  end
-end

lib/gitlab/ci/yaml_processor.rb

@@ -27,7 +27,7 @@ module Gitlab
       end
       def build_attributes(name)
-        job = @jobs[name.to_sym] || {}
+        job = @jobs.fetch(name.to_sym, {})
         { stage_idx: @stages.index(job[:stage]),
           stage: job[:stage],
@@ -53,30 +53,24 @@ module Gitlab
           }.compact }
       end
-      def pipeline_stage_builds(stage, pipeline)
-        selected_jobs = @jobs.select do |_, job|
-          next unless job[:stage] == stage
-
-          only_specs = Gitlab::Ci::Build::Policy
-            .fabricate(job.fetch(:only, {}))
-          except_specs = Gitlab::Ci::Build::Policy
-            .fabricate(job.fetch(:except, {}))
-
-          only_specs.all? { |spec| spec.satisfied_by?(pipeline) } &&
-            except_specs.none? { |spec| spec.satisfied_by?(pipeline) }
-        end
-
-        selected_jobs.map { |_, job| build_attributes(job[:name]) }
+      def stage_builds_attributes(stage)
+        @jobs.values
+          .select { |job| job[:stage] == stage }
+          .map { |job| build_attributes(job[:name]) }
       end
-      def stage_seeds(pipeline)
-        seeds = @stages.uniq.map do |stage|
-          builds = pipeline_stage_builds(stage, pipeline)
-          Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
-        end
-        seeds.compact
+      def stages_attributes
+        @stages.uniq.map do |stage|
+          seeds = stage_builds_attributes(stage).map do |attributes|
+            job = @jobs.fetch(attributes[:name].to_sym)
+            attributes
+              .merge(only: job.fetch(:only, {}))
+              .merge(except: job.fetch(:except, {}))
+          end
+          { name: stage, index: @stages.index(stage), builds: seeds }
+        end
       end
       def self.validation_message(content)

spec/lib/gitlab/ci/pipeline/chain/create_spec.rb

@@ -5,23 +5,23 @@ describe Gitlab::Ci::Pipeline::Chain::Create do
   set(:user) { create(:user) }
   let(:pipeline) do
-    build(:ci_pipeline_with_one_job, project: project,
-                                     ref: 'master')
+    build(:ci_empty_pipeline, project: project, ref: 'master')
   end
   let(:command) do
     Gitlab::Ci::Pipeline::Chain::Command.new(
-      project: project,
-      current_user: user, seeds_block: nil)
+      project: project, current_user: user)
   end
   let(:step) { described_class.new(pipeline, command) }
-  before do
-    step.perform!
-  end
-
   context 'when pipeline is ready to be saved' do
+    before do
+      pipeline.stages.build(name: 'test', project: project)
+
+      step.perform!
+    end
+
     it 'saves a pipeline' do
       expect(pipeline).to be_persisted
     end
@@ -32,6 +32,7 @@ describe Gitlab::Ci::Pipeline::Chain::Create do
     it 'creates stages' do
       expect(pipeline.reload.stages).to be_one
+      expect(pipeline.stages.first).to be_persisted
     end
   end
@@ -40,6 +41,10 @@ describe Gitlab::Ci::Pipeline::Chain::Create do
       build(:ci_pipeline, project: project, ref: nil)
     end
+    before do
+      step.perform!
+    end
+
     it 'breaks the chain' do
       expect(step.break?).to be true
     end
@@ -49,18 +54,4 @@ describe Gitlab::Ci::Pipeline::Chain::Create do
         .to include /Failed to persist the pipeline/
     end
   end
-
-  context 'when there is a seed block present' do
-    let(:seeds) { spy('pipeline seeds') }
-
-    let(:command) do
-      double('command', project: project,
-                        current_user: user,
-                        seeds_block: seeds)
-    end
-
-    it 'executes the block' do
-      expect(seeds).to have_received(:call).with(pipeline)
-    end
-  end
 end

spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Chain::Populate do
+  set(:project) { create(:project) }
+  set(:user) { create(:user) }
+
+  let(:pipeline) do
+    build(:ci_pipeline_with_one_job, project: project,
+                                     ref: 'master')
+  end
+
+  let(:command) do
+    Gitlab::Ci::Pipeline::Chain::Command.new(
+      project: project,
+      current_user: user,
+      seeds_block: nil)
+  end
+
+  let(:step) { described_class.new(pipeline, command) }
+
+  context 'when pipeline doesn not have seeds block' do
+    before do
+      step.perform!
+    end
+
+    it 'does not persist the pipeline' do
+      expect(pipeline).not_to be_persisted
+    end
+
+    it 'does not break the chain' do
+      expect(step.break?).to be false
+    end
+
+    it 'populates pipeline with stages' do
+      expect(pipeline.stages).to be_one
+      expect(pipeline.stages.first).not_to be_persisted
+    end
+
+    it 'populates pipeline with builds' do
+      expect(pipeline.builds).to be_one
+      expect(pipeline.builds.first).not_to be_persisted
+      expect(pipeline.stages.first.builds).to be_one
+      expect(pipeline.stages.first.builds.first).not_to be_persisted
+    end
+  end
+
+  context 'when pipeline is empty' do
+    let(:config) do
+      { rspec: {
+          script: 'ls',
+          only: ['something']
+      } }
+    end
+
+    let(:pipeline) do
+      build(:ci_pipeline, project: project, config: config)
+    end
+
+    before do
+      step.perform!
+    end
+
+    it 'breaks the chain' do
+      expect(step.break?).to be true
+    end
+
+    it 'appends an error about missing stages' do
+      expect(pipeline.errors.to_a)
+        .to include 'No stages / jobs for this pipeline.'
+    end
+  end
+
+  context 'when pipeline has validation errors' do
+    let(:pipeline) do
+      build(:ci_pipeline, project: project, ref: nil)
+    end
+
+    before do
+      step.perform!
+    end
+
+    it 'breaks the chain' do
+      expect(step.break?).to be true
+    end
+
+    it 'appends validation error' do
+      expect(pipeline.errors.to_a)
+        .to include 'Failed to build the pipeline!'
+    end
+  end
+
+  context 'when there is a seed blocks present' do
+    let(:command) do
+      Gitlab::Ci::Pipeline::Chain::Command.new(
+        project: project,
+        current_user: user,
+        seeds_block: seeds_block)
+    end
+
+    context 'when seeds block builds some resources' do
+      let(:seeds_block) do
+        ->(pipeline) { pipeline.variables.build(key: 'VAR', value: '123') }
+      end
+
+      it 'populates pipeline with resources described in the seeds block' do
+        step.perform!
+
+        expect(pipeline).not_to be_persisted
+        expect(pipeline.variables).not_to be_empty
+        expect(pipeline.variables.first).not_to be_persisted
+        expect(pipeline.variables.first.key).to eq 'VAR'
+        expect(pipeline.variables.first.value).to eq '123'
+      end
+    end
+
+    context 'when seeds block tries to persist some resources' do
+      let(:seeds_block) do
+        ->(pipeline) { pipeline.variables.create!(key: 'VAR', value: '123') }
+      end
+
+      it 'raises exception' do
+        expect { step.perform! }.to raise_error(ActiveRecord::RecordNotSaved)
+      end
+    end
+  end
+
+  context 'when pipeline gets persisted during the process' do
+    let(:pipeline) { create(:ci_pipeline, project: project) }
+
+    it 'raises error' do
+      expect { step.perform! }.to raise_error(described_class::PopulateError)
+    end
+  end
+
+  context 'when using only/except build policies' do
+    let(:config) do
+      { rspec: { script: 'rspec', stage: 'test', only: ['master'] },
+        prod: { script: 'cap prod', stage: 'deploy', only: ['tags'] } }
+    end
+
+    let(:pipeline) do
+      build(:ci_pipeline, ref: 'master', config: config)
+    end
+
+    it 'populates pipeline according to used policies' do
+      step.perform!
+
+      expect(pipeline.stages.size).to eq 1
+      expect(pipeline.builds.size).to eq 1
+      expect(pipeline.builds.first.name).to eq 'rspec'
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/chain/validate/config_spec.rb

@@ -76,28 +76,6 @@ describe Gitlab::Ci::Pipeline::Chain::Validate::Config do
     end
   end
-  context 'when pipeline has no stages / jobs' do
-    let(:config) do
-      { rspec: {
-          script: 'ls',
-          only: ['something']
-      } }
-    end
-
-    let(:pipeline) do
-      build(:ci_pipeline, project: project, config: config)
-    end
-
-    it 'appends an error about missing stages' do
-      expect(pipeline.errors.to_a)
-        .to include 'No stages / jobs for this pipeline.'
-    end
-
-    it 'breaks the chain' do
-      expect(step.break?).to be true
-    end
-  end
-
   context 'when pipeline contains configuration validation errors' do
     let(:config) { { rspec: {} } }

spec/lib/gitlab/ci/pipeline/seed/build_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Seed::Build do
+  let(:pipeline) { create(:ci_empty_pipeline) }
+
+  let(:attributes) do
+    { name: 'rspec',
+      ref: 'master',
+      commands: 'rspec' }
+  end
+
+  subject do
+    described_class.new(pipeline, attributes)
+  end
+
+  describe '#attributes' do
+    it 'returns hash attributes of a build' do
+      expect(subject.attributes).to be_a Hash
+      expect(subject.attributes)
+        .to include(:name, :project, :ref, :commands)
+    end
+  end
+
+  describe '#user=' do
+    let(:user) { build(:user) }
+
+    it 'assignes user to a build' do
+      subject.user = user
+
+      expect(subject.attributes).to include(user: user)
+    end
+  end
+
+  describe '#to_resource' do
+    it 'returns a valid build resource' do
+      expect(subject.to_resource).to be_a(::Ci::Build)
+      expect(subject.to_resource).to be_valid
+    end
+
+    it 'memoizes a resource object' do
+      build = subject.to_resource
+
+      expect(build.object_id).to eq subject.to_resource.object_id
+    end
+
+    it 'can not be persisted without explicit assignment' do
+      build = subject.to_resource
+
+      pipeline.save!
+
+      expect(build).not_to be_persisted
+    end
+  end
+
+  describe 'applying only/except policies' do
+    context 'when no branch policy is specified' do
+      let(:attributes) { { name: 'rspec' } }
+
+      it { is_expected.to be_included }
+    end
+
+    context 'when branch policy does not match' do
+      context 'when using only' do
+        let(:attributes) { { name: 'rspec', only: { refs: ['deploy'] } } }
+
+        it { is_expected.not_to be_included }
+      end
+
+      context 'when using except' do
+        let(:attributes) { { name: 'rspec', except: { refs: ['deploy'] } } }
+
+        it { is_expected.to be_included }
+      end
+    end
+
+    context 'when branch regexp policy does not match' do
+      context 'when using only' do
+        let(:attributes) { { name: 'rspec', only: { refs: ['/^deploy$/'] } } }
+
+        it { is_expected.not_to be_included }
+      end
+
+      context 'when using except' do
+        let(:attributes) { { name: 'rspec', except: { refs: ['/^deploy$/'] } } }
+
+        it { is_expected.to be_included }
+      end
+    end
+
+    context 'when branch policy matches' do
+      context 'when using only' do
+        let(:attributes) { { name: 'rspec', only: { refs: %w[deploy master] } } }
+
+        it { is_expected.to be_included }
+      end
+
+      context 'when using except' do
+        let(:attributes) { { name: 'rspec', except: { refs: %w[deploy master] } } }
+
+        it { is_expected.not_to be_included }
+      end
+    end
+
+    context 'when keyword policy matches' do
+      context 'when using only' do
+        let(:attributes) { { name: 'rspec', only: { refs: ['branches'] } } }
+
+        it { is_expected.to be_included }
+      end
+
+      context 'when using except' do
+        let(:attributes) { { name: 'rspec', except: { refs: ['branches'] } } }
+
+        it { is_expected.not_to be_included }
+      end
+    end
+
+    context 'when keyword policy does not match' do
+      context 'when using only' do
+        let(:attributes) { { name: 'rspec', only: { refs: ['tags'] } } }
+
+        it { is_expected.not_to be_included }
+      end
+
+      context 'when using except' do
+        let(:attributes) { { name: 'rspec', except: { refs: ['tags'] } } }
+
+        it { is_expected.to be_included }
+      end
+    end
+
+    context 'when keywords and pipeline source policy matches' do
+      possibilities = [%w[pushes push],
+                       %w[web web],
+                       %w[triggers trigger],
+                       %w[schedules schedule],
+                       %w[api api],
+                       %w[external external]]
+
+      context 'when using only' do
+        possibilities.each do |keyword, source|
+          context "when using keyword `#{keyword}` and source `#{source}`" do
+            let(:pipeline) do
+              build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source)
+            end
+
+            let(:attributes) { { name: 'rspec', only: { refs: [keyword] } } }
+
+            it { is_expected.to be_included }
+          end
+        end
+      end
+
+      context 'when using except' do
+        possibilities.each do |keyword, source|
+          context "when using keyword `#{keyword}` and source `#{source}`" do
+            let(:pipeline) do
+              build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source)
+            end
+
+            let(:attributes) { { name: 'rspec', except: { refs: [keyword] } } }
+
+            it { is_expected.not_to be_included }
+          end
+        end
+      end
+    end
+
+    context 'when keywords and pipeline source does not match' do
+      possibilities = [%w[pushes web],
+                       %w[web push],
+                       %w[triggers schedule],
+                       %w[schedules external],
+                       %w[api trigger],
+                       %w[external api]]
+
+      context 'when using only' do
+        possibilities.each do |keyword, source|
+          context "when using keyword `#{keyword}` and source `#{source}`" do
+            let(:pipeline) do
+              build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source)
+            end
+
+            let(:attributes) { { name: 'rspec', only: { refs: [keyword] } } }
+
+            it { is_expected.not_to be_included }
+          end
+        end
+      end
+
+      context 'when using except' do
+        possibilities.each do |keyword, source|
+          context "when using keyword `#{keyword}` and source `#{source}`" do
+            let(:pipeline) do
+              build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source)
+            end
+
+            let(:attributes) { { name: 'rspec', except: { refs: [keyword] } } }
+
+            it { is_expected.to be_included }
+          end
+        end
+      end
+    end
+
+    context 'when repository path matches' do
+      context 'when using only' do
+        let(:attributes) do
+          { name: 'rspec', only: { refs: ["branches@#{pipeline.project_full_path}"] } }
+        end
+
+        it { is_expected.to be_included }
+      end
+
+      context 'when using except' do
+        let(:attributes) do
+          { name: 'rspec', except: { refs: ["branches@#{pipeline.project_full_path}"] } }
+        end
+
+        it { is_expected.not_to be_included }
+      end
+    end
+
+    context 'when repository path does not matches' do
+      context 'when using only' do
+        let(:attributes) do
+          { name: 'rspec', only: { refs: ['branches@fork'] } }
+        end
+
+        it { is_expected.not_to be_included }
+      end
+
+      context 'when using except' do
+        let(:attributes) do
+          { name: 'rspec', except: { refs: ['branches@fork'] } }
+        end
+
+        it { is_expected.to be_included }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/stage/seed_spec.rb → spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb

 require 'spec_helper'
-describe Gitlab::Ci::Stage::Seed do
+describe Gitlab::Ci::Pipeline::Seed::Stage do
   let(:pipeline) { create(:ci_empty_pipeline) }
-  let(:builds) do
-    [{ name: 'rspec' }, { name: 'spinach' }]
+  let(:attributes) do
+    { name: 'test',
+      index: 0,
+      builds: [{ name: 'rspec' },
+               { name: 'spinach' },
+               { name: 'deploy', only: { refs: ['feature'] } }] }
   end
   subject do
-    described_class.new(pipeline, 'test', builds)
+    described_class.new(pipeline, attributes)
   end
   describe '#size' do
@@ -17,20 +21,46 @@ describe Gitlab::Ci::Stage::Seed do
     end
   end
-  describe '#stage' do
+  describe '#attributes' do
     it 'returns hash attributes of a stage' do
-      expect(subject.stage).to be_a Hash
-      expect(subject.stage).to include(:name, :project)
+      expect(subject.attributes).to be_a Hash
+      expect(subject.attributes).to include(:name, :project)
     end
   end
-  describe '#builds' do
-    it 'returns hash attributes of all builds' do
-      expect(subject.builds.size).to eq 2
-      expect(subject.builds).to all(include(ref: 'master'))
-      expect(subject.builds).to all(include(tag: false))
-      expect(subject.builds).to all(include(project: pipeline.project))
-      expect(subject.builds)
+  describe '#included?' do
+    context 'when it contains builds seeds' do
+      let(:attributes) do
+        { name: 'test',
+          index: 0,
+          builds: [{ name: 'deploy', only: { refs: ['master'] } }] }
+      end
+
+      it { is_expected.to be_included }
+    end
+
+    context 'when it does not contain build seeds' do
+      let(:attributes) do
+        { name: 'test',
+          index: 0,
+          builds: [{ name: 'deploy', only: { refs: ['feature'] } }] }
+      end
+
+      it { is_expected.not_to be_included }
+    end
+  end
+
+  describe '#seeds' do
+    it 'returns build seeds' do
+      expect(subject.seeds).to all(be_a Gitlab::Ci::Pipeline::Seed::Build)
+    end
+
+    it 'returns build seeds including valid attributes' do
+      expect(subject.seeds.size).to eq 2
+      expect(subject.seeds.map(&:attributes)).to all(include(ref: 'master'))
+      expect(subject.seeds.map(&:attributes)).to all(include(tag: false))
+      expect(subject.seeds.map(&:attributes)).to all(include(project: pipeline.project))
+      expect(subject.seeds.map(&:attributes))
         .to all(include(trigger_request: pipeline.trigger_requests.first))
     end
@@ -40,17 +70,27 @@ describe Gitlab::Ci::Stage::Seed do
       end
       it 'returns protected builds' do
-        expect(subject.builds).to all(include(protected: true))
+        expect(subject.seeds.map(&:attributes)).to all(include(protected: true))
       end
     end
-    context 'when a ref is unprotected' do
+    context 'when a ref is not protected' do
       before do
         allow_any_instance_of(Project).to receive(:protected_for?).and_return(false)
       end
       it 'returns unprotected builds' do
-        expect(subject.builds).to all(include(protected: false))
+        expect(subject.seeds.map(&:attributes)).to all(include(protected: false))
+      end
+    end
+
+    it 'filters seeds using only/except policies' do
+      expect(subject.seeds.map(&:attributes)).to satisfy do |seeds|
+        seeds.any? { |hash| hash.fetch(:name) == 'rspec' }
+      end
+
+      expect(subject.seeds.map(&:attributes)).not_to satisfy do |seeds|
+        seeds.any? { |hash| hash.fetch(:name) == 'deploy' }
       end
     end
   end
@@ -61,13 +101,13 @@ describe Gitlab::Ci::Stage::Seed do
     it 'assignes relevant pipeline attributes' do
       subject.user = user
-      expect(subject.builds).to all(include(user: user))
+      expect(subject.seeds.map(&:attributes)).to all(include(user: user))
     end
   end
-  describe '#create!' do
-    it 'creates all stages and builds' do
-      subject.create!
+  describe '#to_resource' do
+    it 'builds a valid stage object with all builds' do
+      subject.to_resource.save!
       expect(pipeline.reload.stages.count).to eq 1
       expect(pipeline.reload.builds.count).to eq 2
@@ -79,5 +119,15 @@ describe Gitlab::Ci::Stage::Seed do
       expect(pipeline.stages)
         .to all(satisfy { |stage| stage.project.present? })
     end
+
+    it 'can not be persisted without explicit pipeline assignment' do
+      stage = subject.to_resource
+
+      pipeline.save!
+
+      expect(stage).not_to be_persisted
+      expect(pipeline.reload.stages.count).to eq 0
+      expect(pipeline.reload.builds.count).to eq 0
+    end
   end
 end

spec/models/ci/pipeline_spec.rb

@@ -177,6 +177,24 @@ describe Ci::Pipeline, :mailer do
     end
   end
+  describe '#protected_ref?' do
+    it 'delegates method to project' do
+      expect(pipeline).not_to be_protected_ref
+    end
+  end
+
+  describe '#legacy_trigger' do
+    let(:trigger_request) { create(:ci_trigger_request) }
+
+    before do
+      pipeline.trigger_requests << trigger_request
+    end
+
+    it 'returns first trigger request' do
+      expect(pipeline.legacy_trigger).to eq trigger_request
+    end
+  end
+
   describe '#auto_canceled?' do
     subject { pipeline.auto_canceled? }
@@ -215,142 +233,257 @@ describe Ci::Pipeline, :mailer do
   end
   describe 'pipeline stages' do
-    before do
-      create(:commit_status, pipeline: pipeline,
-                             stage: 'build',
-                             name: 'linux',
-                             stage_idx: 0,
-                             status: 'success')
-
-      create(:commit_status, pipeline: pipeline,
-                             stage: 'build',
-                             name: 'mac',
-                             stage_idx: 0,
-                             status: 'failed')
-
-      create(:commit_status, pipeline: pipeline,
-                             stage: 'deploy',
-                             name: 'staging',
-                             stage_idx: 2,
-                             status: 'running')
-
-      create(:commit_status, pipeline: pipeline,
-                             stage: 'test',
-                             name: 'rspec',
-                             stage_idx: 1,
-                             status: 'success')
-    end
-
     describe '#stage_seeds' do
-      let(:pipeline) do
-        build(:ci_pipeline, config: { rspec: { script: 'rake' } })
-      end
+      let(:pipeline) { build(:ci_pipeline, config: config) }
+      let(:config) { { rspec: { script: 'rake' } } }
       it 'returns preseeded stage seeds object' do
-        expect(pipeline.stage_seeds).to all(be_a Gitlab::Ci::Stage::Seed)
+        expect(pipeline.stage_seeds)
+          .to all(be_a Gitlab::Ci::Pipeline::Seed::Base)
         expect(pipeline.stage_seeds.count).to eq 1
       end
-    end
-    describe '#seeds_size' do
-      let(:pipeline) { build(:ci_pipeline_with_one_job) }
-      it 'returns number of jobs in stage seeds' do
-        expect(pipeline.seeds_size).to eq 1
+      context 'when no refs policy is specified' do
+        let(:config) do
+          { production: { stage: 'deploy', script: 'cap prod' },
+            rspec: { stage: 'test', script: 'rspec' },
+            spinach: { stage: 'test', script: 'spinach' } }
+        end
+        it 'correctly fabricates a stage seeds object' do
+          seeds = pipeline.stage_seeds
+
+          expect(seeds.size).to eq 2
+          expect(seeds.first.attributes[:name]).to eq 'test'
+          expect(seeds.second.attributes[:name]).to eq 'deploy'
+          expect(seeds.dig(0, 0, :name)).to eq 'rspec'
+          expect(seeds.dig(0, 1, :name)).to eq 'spinach'
+          expect(seeds.dig(1, 0, :name)).to eq 'production'
+        end
       end
-    end
-    describe '#legacy_stages' do
-      subject { pipeline.legacy_stages }
-      context 'stages list' do
-        it 'returns ordered list of stages' do
-          expect(subject.map(&:name)).to eq(%w[build test deploy])
+      context 'when refs policy is specified' do
+        let(:pipeline) do
+          build(:ci_pipeline, ref: 'feature', tag: true, config: config)
+        end
+        let(:config) do
+          { production: { stage: 'deploy', script: 'cap prod', only: ['master'] },
+            spinach: { stage: 'test', script: 'spinach', only: ['tags'] } }
+        end
+
+        it 'returns stage seeds only assigned to master to master' do
+          seeds = pipeline.stage_seeds
+
+          expect(seeds.size).to eq 1
+          expect(seeds.first.attributes[:name]).to eq 'test'
+          expect(seeds.dig(0, 0, :name)).to eq 'spinach'
         end
       end
-      context 'stages with statuses' do
-        let(:statuses) do
-          subject.map { |stage| [stage.name, stage.status] }
+      context 'when source policy is specified' do
+        let(:pipeline) { build(:ci_pipeline, source: :schedule, config: config) }
+
+        let(:config) do
+          { production: { stage: 'deploy', script: 'cap prod', only: ['triggers'] },
+            spinach: { stage: 'test', script: 'spinach', only: ['schedules'] } }
         end
-        it 'returns list of stages with correct statuses' do
-          expect(statuses).to eq([%w(build failed),
-                                  %w(test success),
-                                  %w(deploy running)])
+        it 'returns stage seeds only assigned to schedules' do
+          seeds = pipeline.stage_seeds
+
+          expect(seeds.size).to eq 1
+          expect(seeds.first.attributes[:name]).to eq 'test'
+          expect(seeds.dig(0, 0, :name)).to eq 'spinach'
         end
-        context 'when commit status is retried' do
-          before do
-            create(:commit_status, pipeline: pipeline,
-                                   stage: 'build',
-                                   name: 'mac',
-                                   stage_idx: 0,
-                                   status: 'success')
-            pipeline.process!
+      end
+      context 'when kubernetes policy is specified' do
+        let(:config) do
+          {
+            spinach: { stage: 'test', script: 'spinach' },
+            production: {
+              stage: 'deploy',
+              script: 'cap',
+              only: { kubernetes: 'active' }
+            }
+          }
+        end
+
+        context 'when kubernetes is active' do
+          shared_examples 'same behavior between KubernetesService and Platform::Kubernetes' do
+            it 'returns seeds for kubernetes dependent job' do
+              seeds = pipeline.stage_seeds
+              expect(seeds.size).to eq 2
+              expect(seeds.dig(0, 0, :name)).to eq 'spinach'
+              expect(seeds.dig(1, 0, :name)).to eq 'production'
+            end
           end
-          it 'ignores the previous state' do
-            expect(statuses).to eq([%w(build success),
-                                    %w(test success),
-                                    %w(deploy running)])
+          context 'when user configured kubernetes from Integration > Kubernetes' do
+            let(:project) { create(:kubernetes_project) }
+            let(:pipeline) { build(:ci_pipeline, project: project, config: config) }
+
+            it_behaves_like 'same behavior between KubernetesService and Platform::Kubernetes'
           end
-        end
-      end
-      context 'when there is a stage with warnings' do
-        before do
-          create(:commit_status, pipeline: pipeline,
-                                 stage: 'deploy',
-                                 name: 'prod:2',
-                                 stage_idx: 2,
-                                 status: 'failed',
-                                 allow_failure: true)
+          context 'when user configured kubernetes from CI/CD > Clusters' do
+            let!(:cluster) { create(:cluster, :project, :provided_by_gcp) }
+            let(:project) { cluster.project }
+            let(:pipeline) { build(:ci_pipeline, project: project, config: config) }
+
+            it_behaves_like 'same behavior between KubernetesService and Platform::Kubernetes'
+          end
         end
-        it 'populates stage with correct number of warnings' do
-          deploy_stage = pipeline.legacy_stages.third
-          expect(deploy_stage).not_to receive(:statuses)
-          expect(deploy_stage).to have_warnings
+        context 'when kubernetes is not active' do
+          it 'does not return seeds for kubernetes dependent job' do
+            seeds = pipeline.stage_seeds
+            expect(seeds.size).to eq 1
+            expect(seeds.dig(0, 0, :name)).to eq 'spinach'
+          end
         end
       end
     end
-    describe '#stages_count' do
-      it 'returns a valid number of stages' do
-        expect(pipeline.stages_count).to eq(3)
-      end
-    end
-    describe '#stages_names' do
-      it 'returns a valid names of stages' do
-        expect(pipeline.stages_names).to eq(%w(build test deploy))
+    describe '#seeds_size' do
+      context 'when refs policy is specified' do
+        let(:config) do
+          { production: { stage: 'deploy', script: 'cap prod', only: ['master'] },
+            spinach: { stage: 'test', script: 'spinach', only: ['tags'] } }
+        end
+        let(:pipeline) do
+          build(:ci_pipeline, ref: 'feature', tag: true, config: config)
+        end
+
+        it 'returns real seeds size' do
+          expect(pipeline.seeds_size).to eq 1
+        end
       end
     end
-  end
-
-  describe '#legacy_stage' do
-    subject { pipeline.legacy_stage('test') }
-    context 'with status in stage' do
+    describe 'legacy stages' do
       before do
-        create(:commit_status, pipeline: pipeline, stage: 'test')
+        create(:commit_status, pipeline: pipeline,
+                               stage: 'build',
+                               name: 'linux',
+                               stage_idx: 0,
+                               status: 'success')
+
+        create(:commit_status, pipeline: pipeline,
+                               stage: 'build',
+                               name: 'mac',
+                               stage_idx: 0,
+                               status: 'failed')
+
+        create(:commit_status, pipeline: pipeline,
+                               stage: 'deploy',
+                               name: 'staging',
+                               stage_idx: 2,
+                               status: 'running')
+
+        create(:commit_status, pipeline: pipeline,
+                               stage: 'test',
+                               name: 'rspec',
+                               stage_idx: 1,
+                               status: 'success')
+      end
+
+      describe '#legacy_stages' do
+        subject { pipeline.legacy_stages }
+
+        context 'stages list' do
+          it 'returns ordered list of stages' do
+            expect(subject.map(&:name)).to eq(%w[build test deploy])
+          end
+        end
+
+        context 'stages with statuses' do
+          let(:statuses) do
+            subject.map { |stage| [stage.name, stage.status] }
+          end
+
+          it 'returns list of stages with correct statuses' do
+            expect(statuses).to eq([%w(build failed),
+                                    %w(test success),
+                                    %w(deploy running)])
+          end
+
+          context 'when commit status is retried' do
+            before do
+              create(:commit_status, pipeline: pipeline,
+                                     stage: 'build',
+                                     name: 'mac',
+                                     stage_idx: 0,
+                                     status: 'success')
+
+              pipeline.process!
+            end
+
+            it 'ignores the previous state' do
+              expect(statuses).to eq([%w(build success),
+                                      %w(test success),
+                                      %w(deploy running)])
+            end
+          end
+        end
+
+        context 'when there is a stage with warnings' do
+          before do
+            create(:commit_status, pipeline: pipeline,
+                                   stage: 'deploy',
+                                   name: 'prod:2',
+                                   stage_idx: 2,
+                                   status: 'failed',
+                                   allow_failure: true)
+          end
+
+          it 'populates stage with correct number of warnings' do
+            deploy_stage = pipeline.legacy_stages.third
+
+            expect(deploy_stage).not_to receive(:statuses)
+            expect(deploy_stage).to have_warnings
+          end
+        end
+      end
+
+      describe '#stages_count' do
+        it 'returns a valid number of stages' do
+          expect(pipeline.stages_count).to eq(3)
+        end
       end
-      it { expect(subject).to be_a Ci::LegacyStage }
-      it { expect(subject.name).to eq 'test' }
-      it { expect(subject.statuses).not_to be_empty }
+      describe '#stages_names' do
+        it 'returns a valid names of stages' do
+          expect(pipeline.stages_names).to eq(%w(build test deploy))
+        end
+      end
     end
-    context 'without status in stage' do
-      before do
-        create(:commit_status, pipeline: pipeline, stage: 'build')
+    describe '#legacy_stage' do
+      subject { pipeline.legacy_stage('test') }
+
+      context 'with status in stage' do
+        before do
+          create(:commit_status, pipeline: pipeline, stage: 'test')
+        end
+
+        it { expect(subject).to be_a Ci::LegacyStage }
+        it { expect(subject.name).to eq 'test' }
+        it { expect(subject.statuses).not_to be_empty }
       end
-      it 'return stage object' do
-        is_expected.to be_nil
+      context 'without status in stage' do
+        before do
+          create(:commit_status, pipeline: pipeline, stage: 'build')
+        end
+
+        it 'return stage object' do
+          is_expected.to be_nil
+        end
       end
     end
   end
@@ -589,20 +722,6 @@ describe Ci::Pipeline, :mailer do
     end
   end
-  describe '#has_stage_seeds?' do
-    context 'when pipeline has stage seeds' do
-      subject { build(:ci_pipeline_with_one_job) }
-
-      it { is_expected.to have_stage_seeds }
-    end
-
-    context 'when pipeline does not have stage seeds' do
-      subject { create(:ci_pipeline_without_jobs) }
-
-      it { is_expected.not_to have_stage_seeds }
-    end
-  end
-
   describe '#has_warnings?' do
     subject { pipeline.has_warnings? }

spec/views/ci/lints/show.html.haml_spec.rb

@@ -5,6 +5,7 @@ describe 'ci/lints/show' do
   describe 'XSS protection' do
     let(:config_processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(content)) }
+
     before do
       assign(:status, true)
       assign(:builds, config_processor.builds)