Add latest changes from gitlab-org/security/gitlab@12-6-stable-ee

changelogs/unreleased/security-update-excon-cve-2019-16779.yml

+---
+title: Update excon to 0.71.1 to fix CVE-2019-16779
+merge_request:
+author:
+type: security

changelogs/unreleased/security-workhorse-package-bypass-12-6.yml

+---
+title: Add workhorse request verification to package upload endpoints
+merge_request:
+author:
+type: security

doc/user/asciidoc.md

@@ -221,6 +221,11 @@ include::basics.adoc[]
 include::https://example.org/installation.adoc[]
 ```
+To guarantee good system performance and prevent malicious documents causing
+problems, GitLab enforces a **maximum limit** on the number of include directives
+processed in any one document. Currently a total of 32 documents can be
+included, a number that is inclusive of transitive dependencies.
+
 ### Blocks
 ```asciidoc

lib/api/commit_statuses.rb

@@ -85,6 +85,8 @@ module API
             protected: @project.protected_for?(ref))
         end
+        authorize! :update_pipeline, pipeline
+
         status = GenericCommitStatus.running_or_pending.find_or_initialize_by(
           project: @project,
           pipeline: pipeline,

lib/api/helpers.rb

@@ -256,11 +256,21 @@ module API
     end
     def require_gitlab_workhorse!
+      verify_workhorse_api!
+
       unless env['HTTP_GITLAB_WORKHORSE'].present?
         forbidden!('Request should be executed via GitLab Workhorse')
       end
     end
+    def verify_workhorse_api!
+      Gitlab::Workhorse.verify_api_request!(request.headers)
+    rescue => e
+      Gitlab::ErrorTracking.track_exception(e)
+
+      forbidden!
+    end
+
     def require_pages_enabled!
       not_found! unless user_project.pages_available?
     end

lib/banzai/reference_parser/base_parser.rb

@@ -201,12 +201,14 @@ module Banzai
         gather_references(nodes)
       end
-      # Gathers the references for the given HTML nodes.
+      # Gathers the references for the given HTML nodes.  Returns visible
+      # references and a list of nodes which are not visible to the user
       def gather_references(nodes)
         nodes = nodes_user_can_reference(current_user, nodes)
-        nodes = nodes_visible_to_user(current_user, nodes)
-        referenced_by(nodes)
+        visible = nodes_visible_to_user(current_user, nodes)
+        not_visible = nodes - visible
+        { visible: referenced_by(visible), not_visible: not_visible }
       end
       # Returns a Hash containing the projects for a given list of HTML nodes.

lib/gitlab/asciidoc.rb

@@ -11,6 +11,7 @@ module Gitlab
   # the resulting HTML through HTML pipeline filters.
   module Asciidoc
     MAX_INCLUDE_DEPTH = 5
+    MAX_INCLUDES = 32
     DEFAULT_ADOC_ATTRS = {
         'showtitle' => true,
         'sectanchors' => true,
@@ -40,6 +41,7 @@ module Gitlab
                         extensions: extensions }
       context[:pipeline] = :ascii_doc
+      context[:max_includes] = [MAX_INCLUDES, context[:max_includes]].compact.min
       plantuml_setup

lib/gitlab/asciidoc/include_processor.rb

@@ -13,7 +13,9 @@ module Gitlab
         super(logger: Gitlab::AppLogger)
         @context = context
-        @repository = context[:project].try(:repository)
+        @repository = context[:repository] || context[:project].try(:repository)
+        @max_includes = context[:max_includes].to_i
+        @included = []
         # Note: Asciidoctor calls #freeze on extensions, so we can't set new
         # instance variables after initialization.
@@ -28,8 +30,11 @@ module Gitlab
       def include_allowed?(target, reader)
         doc = reader.document
-        return false if doc.attributes.fetch('max-include-depth').to_i < 1
+        max_include_depth = doc.attributes.fetch('max-include-depth').to_i
+
+        return false if max_include_depth < 1
         return false if target_uri?(target)
+        return false if included.size >= max_includes
         true
       end
@@ -62,7 +67,7 @@ module Gitlab
       private
-      attr_accessor :context, :repository, :cache
+      attr_reader :context, :repository, :cache, :max_includes, :included
       # Gets a Blob at a path for a specific revision.
       # This method will check that the Blob exists and contains readable text.
@@ -77,6 +82,8 @@ module Gitlab
         raise 'Blob not found' unless blob
         raise 'File is not readable' unless blob.readable_text?
+        included << filename
+
         blob
       end

lib/gitlab/git/cross_repo_comparer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class CrossRepoComparer
+      attr_reader :source_repo, :target_repo
+
+      def initialize(source_repo, target_repo)
+        @source_repo = source_repo
+        @target_repo = target_repo
+      end
+
+      def compare(source_ref, target_ref, straight:)
+        ensuring_ref_in_source(target_ref) do |target_commit_id|
+          Gitlab::Git::Compare.new(
+            source_repo,
+            target_commit_id,
+            source_ref,
+            straight: straight
+          )
+        end
+      end
+
+      private
+
+      def ensuring_ref_in_source(ref, &blk)
+        return yield ref if source_repo == target_repo
+
+        # If the commit doesn't exist in the target, there's nothing we can do
+        commit_id = target_repo.commit(ref)&.sha
+        return unless commit_id
+
+        # The commit pointed to by ref may exist in the source even when they
+        # are different repositories. This is particularly true of close forks,
+        # but may also be the case if a temporary ref for this comparison has
+        # already been created in the past, and the result hasn't been GC'd yet.
+        return yield commit_id if source_repo.commit(commit_id)
+
+        # Worst case: the commit is not in the source repo so we need to fetch
+        # it. Use a temporary ref and clean up afterwards
+        with_commit_in_source_tmp(commit_id, &blk)
+      end
+
+      # Fetch the ref into source_repo from target_repo, using a temporary ref
+      # name that will be deleted once the method completes. This is a no-op if
+      # fetching the source branch fails
+      def with_commit_in_source_tmp(commit_id, &blk)
+        tmp_ref = "refs/tmp/#{SecureRandom.hex}"
+
+        yield commit_id if source_repo.fetch_source_branch!(target_repo, commit_id, tmp_ref)
+      ensure
+        source_repo.delete_refs(tmp_ref) # best-effort
+      end
+    end
+  end
+end

lib/gitlab/git/repository.rb

@@ -746,29 +746,9 @@ module Gitlab
       end
       def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:)
-        reachable_ref =
-          if source_repository == self
-            source_branch_name
-          else
-            # If a tmp ref was created before for a separate repo comparison (forks),
-            # we're able to short-circuit the tmp ref re-creation:
-            # 1. Take the SHA from the source repo
-            # 2. Read that in the current "target" repo
-            # 3. If that SHA is still known (readable), it means GC hasn't
-            # cleaned it up yet, so we can use it instead re-writing the tmp ref.
-            source_commit_id = source_repository.commit(source_branch_name)&.sha
-            commit(source_commit_id)&.sha if source_commit_id
-          end
-
-        return compare(target_branch_name, reachable_ref, straight: straight) if reachable_ref
-
-        tmp_ref = "refs/tmp/#{SecureRandom.hex}"
-
-        return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref)
-
-        compare(target_branch_name, tmp_ref, straight: straight)
-      ensure
-        delete_refs(tmp_ref) if tmp_ref
+        CrossRepoComparer
+          .new(source_repository, self)
+          .compare(source_branch_name, target_branch_name, straight: straight)
       end
       def write_ref(ref_path, ref, old_ref: nil)
@@ -1045,13 +1025,6 @@ module Gitlab
       private
-      def compare(base_ref, head_ref, straight:)
-        Gitlab::Git::Compare.new(self,
-                                 base_ref,
-                                 head_ref,
-                                 straight: straight)
-      end
-
       def empty_diff_stats
         Gitlab::Git::DiffStatsCollection.new([])
       end

lib/gitlab/reference_extractor.rb

@@ -6,11 +6,16 @@ module Gitlab
     REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project
                     merge_request snippet commit commit_range directly_addressed_user epic).freeze
     attr_accessor :project, :current_user, :author
+    # This counter is increased by a number of references filtered out by
+    # banzai reference exctractor. Note that this counter is stateful and
+    # not idempotent and is increased whenever you call `references`.
+    attr_reader :stateful_not_visible_counter
     def initialize(project, current_user = nil)
       @project = project
       @current_user = current_user
       @references = {}
+      @stateful_not_visible_counter = 0
       super()
     end
@@ -20,11 +25,15 @@ module Gitlab
     end
     def references(type)
-      super(type, project, current_user)
+      refs = super(type, project, current_user)
+      @stateful_not_visible_counter += refs[:not_visible].count
+
+      refs[:visible]
     end
     def reset_memoized_values
       @references = {}
+      @stateful_not_visible_counter = 0
       super()
     end

package.json

@@ -93,6 +93,7 @@
     "jszip": "^3.1.3",
     "jszip-utils": "^0.0.2",
     "katex": "^0.10.0",
+    "lodash": "^4.17.15",
     "marked": "^0.3.12",
     "mermaid": "^8.4.2",
     "monaco-editor": "^0.15.6",

spec/helpers/projects_helper_spec.rb

@@ -935,14 +935,14 @@ describe ProjectsHelper do
       helper.instance_variable_set(:@project, project)
     end
-    subject { helper.grafana_integration_token }
+    subject { helper.grafana_integration_masked_token }
     it { is_expected.to eq(nil) }
     context 'grafana integration exists' do
       let!(:grafana_integration) { create(:grafana_integration, project: project) }
-      it { is_expected.to eq(grafana_integration.token) }
+      it { is_expected.to eq(grafana_integration.masked_token) }
     end
   end

spec/lib/banzai/reference_parser/mentioned_group_parser_spec.rb

@@ -19,7 +19,7 @@ describe Banzai::ReferenceParser::MentionedGroupParser do
         it 'returns empty array' do
           link['data-group'] = project.group.id.to_s
-          expect(subject.gather_references([link])).to eq([])
+          expect_gathered_references(subject.gather_references([link]), [], 1)
         end
       end
@@ -30,7 +30,7 @@ describe Banzai::ReferenceParser::MentionedGroupParser do
         end
         it 'returns groups' do
-          expect(subject.gather_references([link])).to eq([group])
+          expect_gathered_references(subject.gather_references([link]), [group], 0)
         end
       end
@@ -38,7 +38,7 @@ describe Banzai::ReferenceParser::MentionedGroupParser do
         it 'returns an empty Array' do
           link['data-group'] = 'test-non-existing'
-          expect(subject.gather_references([link])).to eq([])
+          expect_gathered_references(subject.gather_references([link]), [], 1)
         end
       end
     end

spec/lib/banzai/reference_parser/mentioned_project_parser_spec.rb

@@ -19,7 +19,7 @@ describe Banzai::ReferenceParser::MentionedProjectParser do
         it 'returns empty Array' do
           link['data-project'] = project.id.to_s
-          expect(subject.gather_references([link])).to eq([])
+          expect_gathered_references(subject.gather_references([link]), [], 1)
         end
       end
@@ -30,7 +30,7 @@ describe Banzai::ReferenceParser::MentionedProjectParser do
         end
         it 'returns an Array of referenced projects' do
-          expect(subject.gather_references([link])).to eq([project])
+          expect_gathered_references(subject.gather_references([link]), [project], 0)
         end
       end
@@ -38,7 +38,7 @@ describe Banzai::ReferenceParser::MentionedProjectParser do
         it 'returns an empty Array' do
           link['data-project'] = 'inexisting-project-id'
-          expect(subject.gather_references([link])).to eq([])
+          expect_gathered_references(subject.gather_references([link]), [], 1)
         end
       end
     end

spec/lib/banzai/reference_parser/mentioned_user_parser_spec.rb

@@ -22,7 +22,7 @@ describe Banzai::ReferenceParser::MentionedUserParser do
         end
         it 'returns empty list of users' do
-          expect(subject.gather_references([link])).to eq([])
+          expect_gathered_references(subject.gather_references([link]), [], 0)
         end
       end
     end
@@ -35,7 +35,7 @@ describe Banzai::ReferenceParser::MentionedUserParser do
         end
         it 'returns empty list of users' do
-          expect(subject.gather_references([link])).to eq([])
+          expect_gathered_references(subject.gather_references([link]), [], 0)
         end
       end
     end
@@ -44,7 +44,7 @@ describe Banzai::ReferenceParser::MentionedUserParser do
       it 'returns an Array of users' do
         link['data-user'] = user.id.to_s
-        expect(subject.referenced_by([link])).to eq([user])
+        expect_gathered_references(subject.gather_references([link]), [user], 0)
       end
     end
   end

spec/lib/banzai/reference_parser/project_parser_spec.rb

@@ -17,7 +17,7 @@ describe Banzai::ReferenceParser::ProjectParser do
         it 'returns an Array of projects' do
           link['data-project'] = project.id.to_s
-          expect(subject.gather_references([link])).to eq([project])
+          expect_gathered_references(subject.gather_references([link]), [project], 0)
         end
       end
@@ -25,7 +25,7 @@ describe Banzai::ReferenceParser::ProjectParser do
         it 'returns an empty Array' do
           link['data-project'] = ''
-          expect(subject.gather_references([link])).to eq([])
+          expect_gathered_references(subject.gather_references([link]), [], 1)
         end
       end
@@ -35,7 +35,7 @@ describe Banzai::ReferenceParser::ProjectParser do
           link['data-project'] = private_project.id.to_s
-          expect(subject.gather_references([link])).to eq([])
+          expect_gathered_references(subject.gather_references([link]), [], 1)
         end
         it 'returns an Array when authorized' do
@@ -43,7 +43,7 @@ describe Banzai::ReferenceParser::ProjectParser do
           link['data-project'] = private_project.id.to_s
-          expect(subject.gather_references([link])).to eq([private_project])
+          expect_gathered_references(subject.gather_references([link]), [private_project], 0)
         end
       end
     end

spec/lib/gitlab/asciidoc/include_processor_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'nokogiri'
+
+describe Gitlab::Asciidoc::IncludeProcessor do
+  let_it_be(:project) { create(:project, :repository) }
+
+  let(:processor_context) do
+    {
+      project: project,
+      max_includes: max_includes,
+      ref: ref
+    }
+  end
+  let(:ref) { project.repository.root_ref }
+  let(:max_includes) { 10 }
+
+  let(:reader) { Asciidoctor::PreprocessorReader.new(document, lines, 'file.adoc') }
+  let(:document) { Asciidoctor::Document.new(lines) }
+
+  subject(:processor) { described_class.new(processor_context) }
+
+  let(:a_blob) { double(:Blob, readable_text?: true, data: a_data) }
+  let(:a_data) { StringIO.new('include::b.adoc[]') }
+
+  let(:lines) { [':max-include-depth: 1000'] + Array.new(10, 'include::a.adoc[]') }
+
+  before do
+    allow(project.repository).to receive(:blob_at).with(ref, 'a.adoc').and_return(a_blob)
+  end
+
+  describe '#include_allowed?' do
+    it 'allows the first include' do
+      expect(processor.send(:include_allowed?, 'foo.adoc', reader)).to be_truthy
+    end
+
+    it 'disallows the Nth + 1 include' do
+      max_includes.times { processor.send(:read_blob, ref, 'a.adoc') }
+
+      expect(processor.send(:include_allowed?, 'foo.adoc', reader)).to be_falsey
+    end
+  end
+end

spec/lib/gitlab/asciidoc_spec.rb

@@ -425,6 +425,24 @@ module Gitlab
           create_file(current_file, "= AsciiDoc\n")
         end
+        def many_includes(target)
+          Array.new(10, "include::#{target}[]").join("\n")
+        end
+
+        context 'cyclic imports' do
+          before do
+            create_file('doc/api/a.adoc', many_includes('b.adoc'))
+            create_file('doc/api/b.adoc', many_includes('a.adoc'))
+          end
+
+          let(:include_path) { 'a.adoc' }
+          let(:requested_path) { 'doc/api/README.md' }
+
+          it 'completes successfully' do
+            is_expected.to include('<p>Include this:</p>')
+          end
+        end
+
         context 'with path to non-existing file' do
           let(:include_path) { 'not-exists.adoc' }

spec/lib/gitlab/git/cross_repo_comparer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Git::CrossRepoComparer do
+  let(:source_project) { create(:project, :repository) }
+  let(:target_project) { create(:project, :repository) }
+
+  let(:source_repo) { source_project.repository.raw_repository }
+  let(:target_repo) { target_project.repository.raw_repository }
+
+  let(:source_branch) { 'feature' }
+  let(:target_branch) { 'master' }
+  let(:straight) { false }
+
+  let(:source_commit) { source_repo.commit(source_branch) }
+  let(:target_commit) { source_repo.commit(target_branch) }
+
+  subject(:result) { described_class.new(source_repo, target_repo).compare(source_branch, target_branch, straight: straight) }
+
+  describe '#compare' do
+    context 'within a single repository' do
+      let(:target_project) { source_project }
+
+      context 'a non-straight comparison' do
+        it 'compares without fetching from another repo' do
+          expect(source_repo).not_to receive(:fetch_source_branch!)
+
+          expect_compare(result, from: source_commit, to: target_commit)
+          expect(result.straight).to eq(false)
+        end
+      end
+
+      context 'a straight comparison' do
+        let(:straight) { true }
+
+        it 'compares without fetching from another repo' do
+          expect(source_repo).not_to receive(:fetch_source_branch!)
+
+          expect_compare(result, from: source_commit, to: target_commit)
+          expect(result.straight).to eq(true)
+        end
+      end
+    end
+
+    context 'across two repositories' do
+      context 'target ref exists in source repo' do
+        it 'compares without fetching from another repo' do
+          expect(source_repo).not_to receive(:fetch_source_branch!)
+          expect(source_repo).not_to receive(:delete_refs)
+
+          expect_compare(result, from: source_commit, to: target_commit)
+        end
+      end
+
+      context 'target ref does not exist in source repo' do
+        it 'compares in the source repo by fetching from the target to a temporary ref' do
+          new_commit_id = create_commit(target_project.owner, target_repo, target_branch)
+          new_commit = target_repo.commit(new_commit_id)
+
+          # This is how the temporary ref is generated
+          expect(SecureRandom).to receive(:hex).at_least(:once).and_return('foo')
+
+          expect(source_repo)
+            .to receive(:fetch_source_branch!)
+            .with(target_repo, new_commit_id, 'refs/tmp/foo')
+            .and_call_original
+
+          expect(source_repo).to receive(:delete_refs).with('refs/tmp/foo').and_call_original
+
+          expect_compare(result, from: source_commit, to: new_commit)
+        end
+      end
+
+      context 'source ref does not exist in source repo' do
+        let(:source_branch) { 'does-not-exist' }
+
+        it 'returns an empty comparison' do
+          expect(source_repo).not_to receive(:fetch_source_branch!)
+          expect(source_repo).not_to receive(:delete_refs)
+
+          expect(result).to be_a(::Gitlab::Git::Compare)
+          expect(result.commits.size).to eq(0)
+        end
+      end
+
+      context 'target ref does not exist in target repo' do
+        let(:target_branch) { 'does-not-exist' }
+
+        it 'returns nil' do
+          expect(source_repo).not_to receive(:fetch_source_branch!)
+          expect(source_repo).not_to receive(:delete_refs)
+
+          is_expected.to be_nil
+        end
+      end
+    end
+  end
+
+  def expect_compare(of, from:, to:)
+    expect(of).to be_a(::Gitlab::Git::Compare)
+    expect(from).to be_a(::Gitlab::Git::Commit)
+    expect(to).to be_a(::Gitlab::Git::Commit)
+
+    expect(of.commits).not_to be_empty
+    expect(of.head).to eq(from)
+    expect(of.base).to eq(to)
+  end
+
+  def create_commit(user, repo, branch)
+    action = { action: :create, file_path: '/FILE', content: 'content' }
+
+    result = repo.multi_action(user, branch_name: branch, message: 'Commit', actions: [action])
+
+    result.newrev
+  end
+end