Add ignore_if_missing key to include in CI YML (#52680)

lib/gitlab/ci/config.rb

@@ -17,6 +17,8 @@ module Gitlab
         raise Config::ConfigError, e.message
       rescue ::Gitlab::Ci::External::Processor::FileError => e
         raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message
+      rescue ::Gitlab::Ci::External::Mapper::IncludeError => e
+        raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message
       end
       def valid?

lib/gitlab/ci/external/file/local.rb

@@ -5,13 +5,14 @@ module Gitlab
     module External
       module File
         class Local < Base
-          attr_reader :location, :project, :sha
+          attr_reader :location, :project, :sha, :ignore_if_missing
           def initialize(location, opts = {})
             super
             @project = opts.fetch(:project)
             @sha = opts.fetch(:sha)
+            @ignore_if_missing = opts.fetch(:ignore_if_missing)
           end
           def content
@@ -25,7 +26,13 @@ module Gitlab
           private
           def fetch_local_content
-            project.repository.blob_data_at(sha, location)
+            content = project.repository.blob_data_at(sha, location)
+
+            if content.nil? && @ignore_if_missing
+              return '{}'
+            end
+
+            content
           end
         end
       end

lib/gitlab/ci/external/mapper.rb

@@ -4,28 +4,56 @@ module Gitlab
   module Ci
     module External
       class Mapper
+        IncludeError = Class.new(StandardError)
+
         def initialize(values, project, sha)
-          @locations = Array(values.fetch(:include, []))
+          @values = values
           @project = project
           @sha = sha
         end
         def process
-          locations.map { |location| build_external_file(location) }
+          included = @values[:include]
+
+          return [] if included.nil?
+
+          if string_or_array_of_strings?(included)
+            included = Array(included).map do |path|
+              {
+                path: path,
+                ignore_if_missing: false
+              }
+            end
+          elsif included.is_a?(Hash)
+            included = [included]
+          end
+
+          included.map {|i| build_external_file(i) }
         end
         private
-        attr_reader :locations, :project, :sha
-
-        def build_external_file(location)
+        def build_external_file(included)
+          location = included.fetch(:path)
           if ::Gitlab::UrlSanitizer.valid?(location)
+            if included.fetch(:ignore_if_missing)
+              raise IncludeError, 'ignore_if_missing must be false or not included for remote files'
+            end
+
             Gitlab::Ci::External::File::Remote.new(location)
           else
-            options = { project: project, sha: sha }
-            Gitlab::Ci::External::File::Local.new(location, options)
+            Gitlab::Ci::External::File::Local.new(
+              location,
+              project: @project,
+              sha: @sha,
+              ignore_if_missing: included.fetch(:ignore_if_missing, false)
+            )
           end
         end
+
+        def string_or_array_of_strings?(value)
+          value.is_a?(String) || (value.is_a?(Array) && value[0].is_a?(String))
+        end
       end
     end
   end

spec/lib/gitlab/ci/config_spec.rb

-require 'fast_spec_helper'
-
-require_dependency 'active_model'
+require 'spec_helper'
 describe Gitlab::Ci::Config do
   let(:config) do
@@ -193,6 +191,70 @@ describe Gitlab::Ci::Config do
       end
     end
+    context 'when ignore_if_missing true' do
+      let(:gitlab_ci_yml) do
+        <<~HEREDOC
+        include:
+          path: #{local_location}
+          ignore_if_missing: true
+
+
+        image: ruby:2.2
+        HEREDOC
+      end
+
+      context 'when remote url' do
+        let(:gitlab_ci_yml) do
+          <<~HEREDOC
+          include:
+            path: #{remote_location}
+            ignore_if_missing: true
+
+
+          image: ruby:2.2
+          HEREDOC
+        end
+
+        it 'raises an error' do
+          expect { config }.to raise_error(
+            ::Gitlab::Ci::YamlProcessor::ValidationError,
+            "ignore_if_missing must be false for remote files"
+          )
+        end
+      end
+
+      context 'when missing' do
+        before do
+          allow(project.repository)
+            .to receive(:blob_data_at)
+            .and_return(nil)
+        end
+
+        it 'does nothing' do
+          expect(config.to_hash).to eq({
+            image: 'ruby:2.2'
+          })
+        end
+      end
+
+      context 'when present' do
+        it 'does is included' do
+          expect(config.to_hash).to eq({
+            image: 'ruby:2.2',
+            before_script: [
+              "apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs", "ruby -v",
+              "which ruby",
+              "gem install bundler --no-ri --no-rdoc",
+              "bundle install --jobs $(nproc)  \"${FLAGS[@]}\""
+            ],
+            rspec: {
+              script: [ 'bundle exec rspec' ]
+            }
+          })
+        end
+      end
+    end
+
     context "when gitlab_ci.yml has invalid 'include' defined"  do
       let(:gitlab_ci_yml) do
         <<~HEREDOC

spec/lib/gitlab/ci/external/mapper_spec.rb

@@ -26,7 +26,7 @@ describe Gitlab::Ci::External::Mapper do
           expect(subject).to be_an(Array)
         end
-        it 'returns File instances' do
+        it 'returns File::Local instances' do
           expect(subject.first).to be_an_instance_of(Gitlab::Ci::External::File::Local)
         end
       end
@@ -40,10 +40,6 @@ describe Gitlab::Ci::External::Mapper do
           }
         end
-        before do
-          WebMock.stub_request(:get, remote_url).to_return(body: file_content)
-        end
-
         it 'returns an array' do
           expect(subject).to be_an(Array)
         end
@@ -54,6 +50,87 @@ describe Gitlab::Ci::External::Mapper do
       end
     end
+    context 'when include is a hash' do
+      context 'when ignore_if_missing is true' do
+        context 'when using a local file' do
+          let(:values) do
+            {
+              include: {
+                path: '/path1',
+                ignore_if_missing: true
+              }
+            }
+          end
+
+          it 'returns expected File::Local instances' do
+            expect(subject.first).to be_an_instance_of(Gitlab::Ci::External::File::Local)
+            expect(subject.first.ignore_if_missing).to eq(true)
+          end
+        end
+
+        context 'when using a remote file' do
+          let(:values) do
+            {
+              include: {
+                path: 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1235/.gitlab-ci-1.yml',
+                ignore_if_missing: true
+              }
+            }
+          end
+
+          it 'should raise IncludeError' do
+            expect { subject }
+              .to raise_error(
+                Gitlab::Ci::External::Mapper::IncludeError,
+                'ignore_if_missing must be false or not included for remote files'
+            )
+          end
+        end
+
+        context 'when using a local file' do
+          let(:values) do
+            {
+              include: '',
+            }
+          end
+        end
+      end
+    end
+
+    context 'when include is an array of hashes' do
+      let(:values) do
+        {
+          include: [
+            {
+              path: '/path1',
+              ignore_if_missing: true
+            },
+            {
+              path: '/path2',
+              ignore_if_missing: false
+            },
+            {
+              path: '/path3'
+            }
+          ]
+        }
+      end
+
+      it 'returns expected File::Local instances' do
+        expect(subject[0]).to be_an_instance_of(Gitlab::Ci::External::File::Local)
+        expect(subject[0].location).to eq('/path1')
+        expect(subject[0].ignore_if_missing).to eq(true)
+
+        expect(subject[1]).to be_an_instance_of(Gitlab::Ci::External::File::Local)
+        expect(subject[1].location).to eq('/path2')
+        expect(subject[1].ignore_if_missing).to eq(false)
+
+        expect(subject[2]).to be_an_instance_of(Gitlab::Ci::External::File::Local)
+        expect(subject[2].location).to eq('/path3')
+        expect(subject[2].ignore_if_missing).to eq(false)
+      end
+    end
+
     context "when 'include' is defined as an array" do
       let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
       let(:values) do
@@ -67,10 +144,6 @@ describe Gitlab::Ci::External::Mapper do
         }
       end
-      before do
-        WebMock.stub_request(:get, remote_url).to_return(body: file_content)
-      end
-
       it 'returns an array' do
         expect(subject).to be_an(Array)
       end