Add latest changes from gitlab-org/gitlab@master

lib/api/entities/discussion.rb

@@ -5,7 +5,7 @@ module API
     class Discussion < Grape::Entity
       expose :id
       expose :individual_note?, as: :individual_note
-      expose :notes, using: Entities::Note
+      expose :notes, using: Entities::NoteWithGitlabEmployeeBadge
     end
   end
 end

lib/api/entities/note_with_gitlab_employee_badge.rb

+# frozen_string_literal: true
+
+module API
+  module Entities
+    class NoteWithGitlabEmployeeBadge < Note
+      expose :author, using: Entities::UserWithGitlabEmployeeBadge
+      expose :resolved_by, using: Entities::UserWithGitlabEmployeeBadge, if: ->(note, options) { note.resolvable? }
+    end
+  end
+end

lib/api/entities/user_with_gitlab_employee_badge.rb

+# frozen_string_literal: true
+
+module API
+  module Entities
+    class UserWithGitlabEmployeeBadge < UserBasic
+      expose :gitlab_employee?, as: :is_gitlab_employee, if: ->(user, options) { ::Feature.enabled?(:gitlab_employee_badge) && user.gitlab_employee? }
+    end
+  end
+end

lib/gitlab/ci/config/entry/reports.rb

@@ -14,7 +14,7 @@ module Gitlab
           ALLOWED_KEYS =
             %i[junit codequality sast dependency_scanning container_scanning
                dast performance license_management license_scanning metrics lsif
-               dotenv].freeze
+               dotenv cobertura].freeze
           attributes ALLOWED_KEYS
@@ -35,6 +35,7 @@ module Gitlab
               validates :metrics, array_of_strings_or_string: true
               validates :lsif, array_of_strings_or_string: true
               validates :dotenv, array_of_strings_or_string: true
+              validates :cobertura, array_of_strings_or_string: true
             end
           end

lib/gitlab/ci/parsers.rb

@@ -9,7 +9,8 @@ module Gitlab
       def self.parsers
         {
-          junit: ::Gitlab::Ci::Parsers::Test::Junit
+          junit: ::Gitlab::Ci::Parsers::Test::Junit,
+          cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura
         }
       end

lib/gitlab/ci/parsers/coverage/cobertura.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Coverage
+        class Cobertura
+          CoberturaParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
+
+          def parse!(xml_data, coverage_report)
+            root = Hash.from_xml(xml_data)
+
+            parse_all(root, coverage_report)
+          rescue Nokogiri::XML::SyntaxError
+            raise CoberturaParserError, "XML parsing failed"
+          rescue
+            raise CoberturaParserError, "Cobertura parsing failed"
+          end
+
+          private
+
+          def parse_all(root, coverage_report)
+            return unless root.present?
+
+            root.each do |key, value|
+              parse_node(key, value, coverage_report)
+            end
+          end
+
+          def parse_node(key, value, coverage_report)
+            if key == 'class'
+              Array.wrap(value).each do |item|
+                parse_class(item, coverage_report)
+              end
+            elsif value.is_a?(Hash)
+              parse_all(value, coverage_report)
+            elsif value.is_a?(Array)
+              value.each do |item|
+                parse_all(item, coverage_report)
+              end
+            end
+          end
+
+          def parse_class(file, coverage_report)
+            return unless file["filename"].present? && file["lines"].present?
+
+            parsed_lines = parse_lines(file["lines"])
+
+            coverage_report.add_file(file["filename"], Hash[parsed_lines])
+          end
+
+          def parse_lines(lines)
+            line_array = Array.wrap(lines["line"])
+
+            line_array.map do |line|
+              # Using `Integer()` here to raise exception on invalid values
+              [Integer(line["number"]), Integer(line["hits"])]
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/reports/coverage_reports.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      class CoverageReports
+        attr_reader :files
+
+        def initialize
+          @files = {}
+        end
+
+        def pick(keys)
+          coverage_files = files.select do |key|
+            keys.include?(key)
+          end
+
+          { files: coverage_files }
+        end
+
+        def add_file(name, line_coverage)
+          if files[name].present?
+            line_coverage.each { |line, hits| combine_lines(name, line, hits) }
+
+          else
+            files[name] = line_coverage
+          end
+        end
+
+        private
+
+        def combine_lines(name, line, hits)
+          if files[name][line].present?
+            files[name][line] += hits
+
+          else
+            files[name][line] = hits
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/group/tree_saver.rb

@@ -49,7 +49,7 @@ module Gitlab
         end
         def tree_saver
-          @tree_saver ||= RelationTreeSaver.new
+          @tree_saver ||= LegacyRelationTreeSaver.new
         end
       end
     end

lib/gitlab/import_export/json/legacy_writer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class LegacyWriter
+        include Gitlab::ImportExport::CommandLineUtil
+
+        attr_reader :path
+
+        def initialize(path)
+          @path = path
+          @last_array = nil
+          @keys = Set.new
+
+          mkdir_p(File.dirname(@path))
+          file.write('{}')
+        end
+
+        def close
+          @file&.close
+          @file = nil
+        end
+
+        def set(hash)
+          hash.each do |key, value|
+            write(key, value)
+          end
+        end
+
+        def write(key, value)
+          raise ArgumentError, "key '#{key}' already written" if @keys.include?(key)
+
+          # rewind by one byte, to overwrite '}'
+          file.pos = file.size - 1
+
+          file.write(',') if @keys.any?
+          file.write(key.to_json)
+          file.write(':')
+          file.write(value.to_json)
+          file.write('}')
+
+          @keys.add(key)
+          @last_array = nil
+          @last_array_count = nil
+        end
+
+        def append(key, value)
+          unless @last_array == key
+            write(key, [])
+
+            @last_array = key
+            @last_array_count = 0
+          end
+
+          # rewind by two bytes, to overwrite ']}'
+          file.pos = file.size - 2
+
+          file.write(',') if @last_array_count > 0
+          file.write(value.to_json)
+          file.write(']}')
+          @last_array_count += 1
+        end
+
+        private
+
+        def file
+          @file ||= File.open(@path, "wb")
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/json/streaming_serializer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class StreamingSerializer
+        include Gitlab::ImportExport::CommandLineUtil
+
+        BATCH_SIZE = 100
+
+        class Raw < String
+          def to_json(*_args)
+            to_s
+          end
+        end
+
+        def initialize(exportable, relations_schema, json_writer)
+          @exportable = exportable
+          @relations_schema = relations_schema
+          @json_writer = json_writer
+        end
+
+        def execute
+          serialize_root
+
+          includes.each do |relation_definition|
+            serialize_relation(relation_definition)
+          end
+        end
+
+        private
+
+        attr_reader :json_writer, :relations_schema, :exportable
+
+        def serialize_root
+          attributes = exportable.as_json(
+            relations_schema.merge(include: nil, preloads: nil))
+          json_writer.set(attributes)
+        end
+
+        def serialize_relation(definition)
+          raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
+          raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
+
+          key, options = definition.first
+
+          record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
+          if record.is_a?(ActiveRecord::Relation)
+            serialize_many_relations(key, record, options)
+          else
+            serialize_single_relation(key, record, options)
+          end
+        end
+
+        def serialize_many_relations(key, records, options)
+          key_preloads = preloads&.dig(key)
+          records = records.preload(key_preloads) if key_preloads
+
+          records.find_each(batch_size: BATCH_SIZE) do |record|
+            json = Raw.new(record.to_json(options))
+
+            json_writer.append(key, json)
+          end
+        end
+
+        def serialize_single_relation(key, record, options)
+          json = Raw.new(record.to_json(options))
+
+          json_writer.write(key, json)
+        end
+
+        def includes
+          relations_schema[:include]
+        end
+
+        def preloads
+          relations_schema[:preload]
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/relation_tree_saver.rb → lib/gitlab/import_export/legacy_relation_tree_saver.rb

@@ -2,7 +2,7 @@
 module Gitlab
   module ImportExport
-    class RelationTreeSaver
+    class LegacyRelationTreeSaver
       include Gitlab::ImportExport::CommandLineUtil
       def serialize(exportable, relations_tree)

lib/gitlab/import_export/project/legacy_tree_saver.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class LegacyTreeSaver
+        attr_reader :full_path
+
+        def initialize(project:, current_user:, shared:, params: {})
+          @params       = params
+          @project      = project
+          @current_user = current_user
+          @shared       = shared
+          @full_path    = File.join(@shared.export_path, ImportExport.project_filename)
+        end
+
+        def save
+          project_tree = tree_saver.serialize(@project, reader.project_tree)
+          fix_project_tree(project_tree)
+          tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
+
+          true
+        rescue => e
+          @shared.error(e)
+          false
+        end
+
+        private
+
+        # Aware that the resulting hash needs to be pure-hash and
+        # does not include any AR objects anymore, only objects that run `.to_json`
+        def fix_project_tree(project_tree)
+          if @params[:description].present?
+            project_tree['description'] = @params[:description]
+          end
+
+          project_tree['project_members'] += group_members_array
+        end
+
+        def reader
+          @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+        end
+
+        def group_members_array
+          group_members.as_json(reader.group_members_tree).each do |group_member|
+            group_member['source_type'] = 'Project' # Make group members project members of the future import
+          end
+        end
+
+        def group_members
+          return [] unless @current_user.can?(:admin_group, @project.group)
+
+          # We need `.where.not(user_id: nil)` here otherwise when a group has an
+          # invitee, it would make the following query return 0 rows since a NULL
+          # user_id would be present in the subquery
+          # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
+          non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
+
+          GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
+        end
+
+        def tree_saver
+          @tree_saver ||= Gitlab::ImportExport::LegacyRelationTreeSaver.new
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/project/tree_saver.rb

@@ -15,52 +15,40 @@ module Gitlab
         end
         def save
-          project_tree = tree_saver.serialize(@project, reader.project_tree)
-          fix_project_tree(project_tree)
-          tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
+          json_writer = ImportExport::JSON::LegacyWriter.new(@full_path)
+
+          serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer)
+          serializer.execute
           true
         rescue => e
           @shared.error(e)
           false
+        ensure
+          json_writer&.close
         end
         private
-        # Aware that the resulting hash needs to be pure-hash and
-        # does not include any AR objects anymore, only objects that run `.to_json`
-        def fix_project_tree(project_tree)
-          if @params[:description].present?
-            project_tree['description'] = @params[:description]
-          end
-
-          project_tree['project_members'] += group_members_array
-        end
-
         def reader
           @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
         end
-        def group_members_array
-          group_members.as_json(reader.group_members_tree).each do |group_member|
-            group_member['source_type'] = 'Project' # Make group members project members of the future import
-          end
+        def exportable
+          @project.present(exportable_params)
         end
-        def group_members
-          return [] unless @current_user.can?(:admin_group, @project.group)
-
-          # We need `.where.not(user_id: nil)` here otherwise when a group has an
-          # invitee, it would make the following query return 0 rows since a NULL
-          # user_id would be present in the subquery
-          # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
-          non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
-
-          GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
+        def exportable_params
+          params = {
+            presenter_class: presenter_class,
+            current_user: @current_user
+          }
+          params[:override_description] = @params[:description] if @params[:description].present?
+          params
         end
-        def tree_saver
-          @tree_saver ||= RelationTreeSaver.new
+        def presenter_class
+          Projects::ImportExport::ProjectExportPresenter
         end
       end
     end

lib/tasks/gitlab/import_export/export.rake

@@ -19,7 +19,6 @@ namespace :gitlab do
         if ENV['EXPORT_DEBUG'].present?
           ActiveRecord::Base.logger = logger
-          Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
           logger.level = Logger::DEBUG
         else
           logger.level = Logger::INFO

lib/tasks/gitlab/import_export/import.rake

@@ -23,7 +23,6 @@ namespace :gitlab do
         if ENV['IMPORT_DEBUG'].present?
           ActiveRecord::Base.logger = logger
-          Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
           logger.level = Logger::DEBUG
         else
           logger.level = Logger::INFO

lib/tasks/sidekiq.rake

@@ -33,6 +33,6 @@ namespace :sidekiq do
   task :launchd do
     deprecation_warning!
-    system(*%w(bin/background_jobs start_no_deamonize))
+    system(*%w(bin/background_jobs start_silent))
   end
 end

locale/gitlab.pot

@@ -1810,6 +1810,9 @@ msgstr ""
 msgid "An error occurred while enabling Service Desk."
 msgstr ""
+msgid "An error occurred while fetching coverage reports."
+msgstr ""
+
 msgid "An error occurred while fetching environments."
 msgstr ""
@@ -4822,6 +4825,9 @@ msgstr ""
 msgid "ClusterIntegration|You must have an RBAC-enabled cluster to install Knative."
 msgstr ""
+msgid "ClusterIntegration|You should select at least two subnets"
+msgstr ""
+
 msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}"
 msgstr ""
@@ -13325,6 +13331,9 @@ msgstr ""
 msgid "No template"
 msgstr ""
+msgid "No test coverage"
+msgstr ""
+
 msgid "No thanks, don't show this again"
 msgstr ""
@@ -19472,6 +19481,11 @@ msgstr ""
 msgid "Test coverage parsing"
 msgstr ""
+msgid "Test coverage: %d hit"
+msgid_plural "Test coverage: %d hits"
+msgstr[0] ""
+msgstr[1] ""
+
 msgid "Test failed."
 msgstr ""

spec/controllers/projects/merge_requests_controller_spec.rb

@@ -984,6 +984,136 @@ describe Projects::MergeRequestsController do
     end
   end
+  describe 'GET coverage_reports' do
+    let(:merge_request) do
+      create(:merge_request,
+        :with_merge_request_pipeline,
+        target_project: project,
+        source_project: project)
+    end
+
+    let(:pipeline) do
+      create(:ci_pipeline,
+        :success,
+        project: merge_request.source_project,
+        ref: merge_request.source_branch,
+        sha: merge_request.diff_head_sha)
+    end
+
+    before do
+      allow_any_instance_of(MergeRequest)
+        .to receive(:find_coverage_reports)
+        .and_return(report)
+
+      allow_any_instance_of(MergeRequest)
+        .to receive(:actual_head_pipeline)
+        .and_return(pipeline)
+    end
+
+    subject do
+      get :coverage_reports, params: {
+        namespace_id: project.namespace.to_param,
+        project_id: project,
+        id: merge_request.iid
+      },
+      format: :json
+    end
+
+    describe 'permissions on a public project with private CI/CD' do
+      let(:project) { create :project, :repository, :public, :builds_private }
+      let(:report) { { status: :parsed, data: [] } }
+
+      context 'while signed out' do
+        before do
+          sign_out(user)
+        end
+
+        it 'responds with a 404' do
+          subject
+
+          expect(response).to have_gitlab_http_status(:not_found)
+          expect(response.body).to be_blank
+        end
+      end
+
+      context 'while signed in as an unrelated user' do
+        before do
+          sign_in(create(:user))
+        end
+
+        it 'responds with a 404' do
+          subject
+
+          expect(response).to have_gitlab_http_status(:not_found)
+          expect(response.body).to be_blank
+        end
+      end
+    end
+
+    context 'when pipeline has jobs with coverage reports' do
+      before do
+        allow_any_instance_of(MergeRequest)
+          .to receive(:has_coverage_reports?)
+          .and_return(true)
+      end
+
+      context 'when processing coverage reports is in progress' do
+        let(:report) { { status: :parsing } }
+
+        it 'sends polling interval' do
+          expect(Gitlab::PollingInterval).to receive(:set_header)
+
+          subject
+        end
+
+        it 'returns 204 HTTP status' do
+          subject
+
+          expect(response).to have_gitlab_http_status(:no_content)
+        end
+      end
+
+      context 'when processing coverage reports is completed' do
+        let(:report) { { status: :parsed, data: pipeline.coverage_reports } }
+
+        it 'returns coverage reports' do
+          subject
+
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(json_response).to eq({ 'files' => {} })
+        end
+      end
+
+      context 'when user created corrupted coverage reports' do
+        let(:report) { { status: :error, status_reason: 'Failed to parse coverage reports' } }
+
+        it 'does not send polling interval' do
+          expect(Gitlab::PollingInterval).not_to receive(:set_header)
+
+          subject
+        end
+
+        it 'returns 400 HTTP status' do
+          subject
+
+          expect(response).to have_gitlab_http_status(:bad_request)
+          expect(json_response).to eq({ 'status_reason' => 'Failed to parse coverage reports' })
+        end
+      end
+    end
+
+    context 'when pipeline does not have jobs with coverage reports' do
+      let(:report) { double }
+
+      it 'returns no content' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:no_content)
+        expect(response.body).to be_empty
+      end
+    end
+  end
+
   describe 'GET test_reports' do
     let(:merge_request) do
       create(:merge_request,

spec/factories/ci/builds.rb

@@ -311,6 +311,12 @@ FactoryBot.define do
       end
     end
+    trait :coverage_reports do
+      after(:build) do |build|
+        build.job_artifacts << create(:ci_job_artifact, :cobertura, job: build)
+      end
+    end
+
     trait :expired do
       artifacts_expire_at { 1.minute.ago }
     end
@@ -355,6 +361,8 @@ FactoryBot.define do
       options { {} }
     end
+    # TODO: move Security traits to ee_ci_build
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/210486
     trait :dast do
       options do
         {
@@ -395,6 +403,14 @@ FactoryBot.define do
       end
     end
+    trait :license_scanning do
+      options do
+        {
+          artifacts: { reports: { license_management: 'gl-license-scanning-report.json' } }
+        }
+      end
+    end
+
     trait :non_playable do
       status { 'created' }
       self.when { 'manual' }

spec/factories/ci/job_artifacts.rb

@@ -129,6 +129,36 @@ FactoryBot.define do
       end
     end
+    trait :cobertura do
+      file_type { :cobertura }
+      file_format { :gzip }
+
+      after(:build) do |artifact, evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/cobertura/coverage.xml.gz'), 'application/x-gzip')
+      end
+    end
+
+    trait :coverage_gocov_xml do
+      file_type { :cobertura }
+      file_format { :gzip }
+
+      after(:build) do |artifact, evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/cobertura/coverage_gocov_xml.xml.gz'), 'application/x-gzip')
+      end
+    end
+
+    trait :coverage_with_corrupted_data do
+      file_type { :cobertura }
+      file_format { :gzip }
+
+      after(:build) do |artifact, evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/cobertura/coverage_with_corrupted_data.xml.gz'), 'application/x-gzip')
+      end
+    end
+
     trait :codequality do
       file_type { :codequality }
       file_format { :raw }