Add latest changes from gitlab-org/gitlab@master

Gemfile

@@ -448,9 +448,9 @@ end
 # Gitaly GRPC protocol definitions
 gem 'gitaly', '~> 1.65.0'
-gem 'grpc', '~> 1.19.0'
-gem 'google-protobuf', '~> 3.7.1'
+gem 'grpc', '~> 1.24.0'
+gem 'google-protobuf', '~> 3.8.0'
 gem 'toml-rb', '~> 1.0.0', require: false

Gemfile.lock

@@ -400,7 +400,7 @@ GEM
       mime-types (~> 3.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
-    google-protobuf (3.7.1)
+    google-protobuf (3.8.0)
     googleapis-common-protos-types (1.0.4)
       google-protobuf (~> 3.0)
     googleauth (0.6.6)
@@ -440,9 +440,9 @@ GEM
       graphql (~> 1.6)
       html-pipeline (~> 2.8)
       sass (~> 3.4)
-    grpc (1.19.0)
-      google-protobuf (~> 3.1)
-      googleapis-common-protos-types (~> 1.0.0)
+    grpc (1.24.0)
+      google-protobuf (~> 3.8)
+      googleapis-common-protos-types (~> 1.0)
     gssapi (1.2.0)
       ffi (>= 1.0.1)
     haml (5.0.4)
@@ -1181,7 +1181,7 @@ DEPENDENCIES
   gitlab_omniauth-ldap (~> 2.1.1)
   gon (~> 6.2)
   google-api-client (~> 0.23)
-  google-protobuf (~> 3.7.1)
+  google-protobuf (~> 3.8.0)
   gpgme (~> 2.0.18)
   grape (~> 1.1.0)
   grape-entity (~> 0.7.1)
@@ -1190,7 +1190,7 @@ DEPENDENCIES
   graphiql-rails (~> 1.4.10)
   graphql (~> 1.9.11)
   graphql-docs (~> 1.6.0)
-  grpc (~> 1.19.0)
+  grpc (~> 1.24.0)
   gssapi
   haml_lint (~> 0.31.0)
   hamlit (~> 2.8.8)

app/controllers/projects/merge_requests_controller.rb

@@ -13,7 +13,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
   skip_before_action :merge_request, only: [:index, :bulk_update]
   before_action :whitelist_query_limiting, only: [:assign_related_issues, :update]
   before_action :authorize_update_issuable!, only: [:close, :edit, :update, :remove_wip, :sort]
-  before_action :authorize_test_reports!, only: [:test_reports]
+  before_action :authorize_read_actual_head_pipeline!, only: [:test_reports, :exposed_artifacts]
   before_action :set_issuables_index, only: [:index]
   before_action :authenticate_user!, only: [:assign_related_issues]
   before_action :check_user_can_push_to_source_branch!, only: [:rebase]
@@ -115,6 +115,14 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
     reports_response(@merge_request.compare_test_reports)
   end
+  def exposed_artifacts
+    if @merge_request.has_exposed_artifacts?
+      reports_response(@merge_request.find_exposed_artifacts)
+    else
+      head :no_content
+    end
+  end
+
   def edit
     define_edit_vars
   end
@@ -357,8 +365,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
     end
   end
-  def authorize_test_reports!
-    # MergeRequest#actual_head_pipeline is the pipeline accessed in MergeRequest#compare_reports.
+  def authorize_read_actual_head_pipeline!
     return render_404 unless can?(current_user, :read_build, merge_request.actual_head_pipeline)
   end
 end

app/helpers/builds_helper.rb

@@ -4,12 +4,12 @@ module BuildsHelper
   def build_summary(build, skip: false)
     if build.has_trace?
       if skip
-        link_to _("View job trace"), pipeline_job_url(build.pipeline, build)
+        link_to _("View job log"), pipeline_job_url(build.pipeline, build)
       else
         build.trace.html(last_lines: 10).html_safe
       end
     else
-      _("No job trace")
+      _("No job log")
     end
   end

app/models/ci/build.rb

@@ -118,6 +118,11 @@ module Ci
     scope :eager_load_job_artifacts, -> { includes(:job_artifacts) }
+    scope :with_exposed_artifacts, -> do
+      joins(:metadata).merge(Ci::BuildMetadata.with_exposed_artifacts)
+        .includes(:metadata, :job_artifacts_metadata)
+    end
+
     scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
     scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) }
     scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
@@ -595,6 +600,14 @@ module Ci
       update_column(:trace, nil)
     end
+    def artifacts_expose_as
+      options.dig(:artifacts, :expose_as)
+    end
+
+    def artifacts_paths
+      options.dig(:artifacts, :paths)
+    end
+
     def needs_touch?
       Time.now - updated_at > 15.minutes.to_i
     end

app/models/ci/build_metadata.rb

@@ -27,6 +27,7 @@ module Ci
     scope :scoped_build, -> { where('ci_builds_metadata.build_id = ci_builds.id') }
     scope :with_interruptible, -> { where(interruptible: true) }
+    scope :with_exposed_artifacts, -> { where(has_exposed_artifacts: true) }
     enum timeout_source: {
         unknown_timeout_source: 1,

app/models/ci/pipeline.rb

@@ -783,6 +783,10 @@ module Ci
       end
     end
+    def has_exposed_artifacts?
+      complete? && builds.latest.with_exposed_artifacts.exists?
+    end
+
     def branch_updated?
       strong_memoize(:branch_updated) do
         push_details.branch_updated?

app/models/concerns/ci/metadatable.rb

@@ -16,6 +16,7 @@ module Ci
       delegate :timeout, to: :metadata, prefix: true, allow_nil: true
       delegate :interruptible, to: :metadata, prefix: false, allow_nil: true
+      delegate :has_exposed_artifacts?, to: :metadata, prefix: false, allow_nil: true
       before_create :ensure_metadata
     end
@@ -45,6 +46,9 @@ module Ci
     def options=(value)
       write_metadata_attribute(:options, :config_options, value)
+
+      # Store presence of exposed artifacts in build metadata to make it easier to query
+      ensure_metadata.has_exposed_artifacts = value&.dig(:artifacts, :expose_as).present?
     end
     def yaml_variables=(value)

app/models/merge_request.rb

@@ -1255,6 +1255,27 @@ class MergeRequest < ApplicationRecord
     compare_reports(Ci::CompareTestReportsService)
   end
+  def has_exposed_artifacts?
+    return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true)
+
+    actual_head_pipeline&.has_exposed_artifacts?
+  end
+
+  # TODO: this method and compare_test_reports use the same
+  # result type, which is handled by the controller's #reports_response.
+  # we should minimize mistakes by isolating the common parts.
+  # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
+  def find_exposed_artifacts
+    unless has_exposed_artifacts?
+      return { status: :error, status_reason: 'This merge request does not have exposed artifacts' }
+    end
+
+    compare_reports(Ci::GenerateExposedArtifactsReportService)
+  end
+
+  # TODO: consider renaming this as with exposed artifacts we generate reports,
+  # not always compare
+  # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
   def compare_reports(service_class, current_user = nil)
     with_reactive_cache(service_class.name, current_user&.id) do |data|
       unless service_class.new(project, current_user)
@@ -1269,6 +1290,8 @@ class MergeRequest < ApplicationRecord
   def calculate_reactive_cache(identifier, current_user_id = nil, *args)
     service_class = identifier.constantize
+    # TODO: the type check should change to something that includes exposed artifacts service
+    # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
     raise NameError, service_class unless service_class < Ci::CompareReportsBaseService
     current_user = User.find_by(id: current_user_id)

app/serializers/merge_request_poll_widget_entity.rb

@@ -65,6 +65,12 @@ class MergeRequestPollWidgetEntity < IssuableEntity
     end
   end
+  expose :exposed_artifacts_path do |merge_request|
+    if merge_request.has_exposed_artifacts?
+      exposed_artifacts_project_merge_request_path(merge_request.project, merge_request, format: :json)
+    end
+  end
+
   expose :create_issue_to_resolve_discussions_path do |merge_request|
     presenter(merge_request).create_issue_to_resolve_discussions_path
   end

app/services/ci/compare_reports_base_service.rb

 # frozen_string_literal: true
 module Ci
+  # TODO: when using this class with exposed artifacts we see that there are
+  # 2 responsibilities:
+  # 1. reactive caching interface (same in all cases)
+  # 2. data generator (report comparison in most of the case but not always)
+  # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
   class CompareReportsBaseService < ::BaseService
     def execute(base_pipeline, head_pipeline)
       comparer = comparer_class.new(get_report(base_pipeline), get_report(head_pipeline))

app/services/ci/find_exposed_artifacts_service.rb

+# frozen_string_literal: true
+
+module Ci
+  # This class loops through all builds with exposed artifacts and returns
+  # basic information about exposed artifacts for given jobs for the frontend
+  # to display them as custom links in the merge request.
+  #
+  # This service must be used with care.
+  # Looking for exposed artifacts is very slow and should be done asynchronously.
+  class FindExposedArtifactsService < ::BaseService
+    include Gitlab::Routing
+
+    MAX_EXPOSED_ARTIFACTS = 10
+
+    def for_pipeline(pipeline, limit: MAX_EXPOSED_ARTIFACTS)
+      results = []
+
+      pipeline.builds.latest.with_exposed_artifacts.find_each do |job|
+        if job_exposed_artifacts = for_job(job)
+          results << job_exposed_artifacts
+        end
+
+        break if results.size >= limit
+      end
+
+      results
+    end
+
+    def for_job(job)
+      return unless job.has_exposed_artifacts?
+
+      metadata_entries = first_2_metadata_entries_for_artifacts_paths(job)
+      return if metadata_entries.empty?
+
+      {
+        text: job.artifacts_expose_as,
+        url: path_for_entries(metadata_entries, job),
+        job_path: project_job_path(project, job),
+        job_name: job.name
+      }
+    end
+
+    private
+
+    # we don't need to fetch all artifacts entries for a job because
+    # it could contain many. We only need to know whether it has 1 or more
+    # artifacts, so fetching the first 2 would be sufficient.
+    def first_2_metadata_entries_for_artifacts_paths(job)
+      job.artifacts_paths
+        .lazy
+        .map { |path| job.artifacts_metadata_entry(path, recursive: true) }
+        .select { |entry| entry.exists? }
+        .first(2)
+    end
+
+    def path_for_entries(entries, job)
+      return if entries.empty?
+
+      if single_artifact?(entries)
+        file_project_job_artifacts_path(project, job, entries.first.path)
+      else
+        browse_project_job_artifacts_path(project, job)
+      end
+    end
+
+    def single_artifact?(entries)
+      entries.size == 1 && entries.first.file?
+    end
+  end
+end

app/services/ci/generate_exposed_artifacts_report_service.rb

+# frozen_string_literal: true
+
+module Ci
+  # TODO: a couple of points with this approach:
+  # + reuses existing architecture and reactive caching
+  # - it's not a report comparison and some comparing features must be turned off.
+  # see CompareReportsBaseService for more notes.
+  # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
+  class GenerateExposedArtifactsReportService < CompareReportsBaseService
+    def execute(base_pipeline, head_pipeline)
+      data = FindExposedArtifactsService.new(project, current_user).for_pipeline(head_pipeline)
+      {
+        status: :parsed,
+        key: key(base_pipeline, head_pipeline),
+        data: data
+      }
+    rescue => e
+      Gitlab::Sentry.track_acceptable_exception(e, extra: { project_id: project.id })
+      {
+        status: :error,
+        key: key(base_pipeline, head_pipeline),
+        status_reason: _('An error occurred while fetching exposed artifacts.')
+      }
+    end
+
+    def latest?(base_pipeline, head_pipeline, data)
+      data&.fetch(:key, nil) == key(base_pipeline, head_pipeline)
+    end
+  end
+end

app/views/projects/_export.html.haml

@@ -14,7 +14,7 @@
           %li= desc
       %p= _('The following items will NOT be exported:')
       %ul
-        %li= _('Job traces and artifacts')
+        %li= _('Job logs and artifacts')
         %li= _('Container registry images')
         %li= _('CI variables')
         %li= _('Webhooks')

app/views/projects/services/_form.html.haml

@@ -6,7 +6,6 @@
         - hide_class = 'd-none' if @service.activated? != value
         %span.js-service-active-status{ class: hide_class, data: { value: value.to_s } }
           = boolean_to_icon value
-    %p= #{@service.description}.
     - if @service.respond_to?(:detailed_description)
       %p= @service.detailed_description

app/views/projects/settings/ci_cd/_form.html.haml

@@ -98,7 +98,7 @@
             %span.input-group-append
               .input-group-text /
           %p.form-text.text-muted
-            = _("A regular expression that will be used to find the test coverage output in the job trace. Leave blank to disable")
+            = _("A regular expression that will be used to find the test coverage output in the job log. Leave blank to disable")
             = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'test-coverage-parsing'), target: '_blank'
           .bs-callout.bs-callout-info
             %p= _("Below are examples of regex for existing tools:")

app/views/projects/show.html.haml

@@ -6,7 +6,7 @@
 = render partial: 'flash_messages', locals: { project: @project }
-- if !@project.empty_repo? && can?(current_user, :download_code, @project)
+- if !@project.empty_repo? && can?(current_user, :download_code, @project) && !vue_file_list_enabled?
   - signatures_path = project_signatures_path(@project, @project.default_branch)
   .js-signature-container{ data: { 'signatures-path': signatures_path } }

app/views/projects/tree/show.html.haml

@@ -6,7 +6,8 @@
 = content_for :meta_tags do
   = auto_discovery_link_tag(:atom, project_commits_url(@project, @ref, rss_url_options), title: "#{@project.name}:#{@ref} commits")
-.js-signature-container{ data: { 'signatures-path': signatures_path } }
+- unless vue_file_list_enabled?
+  .js-signature-container{ data: { 'signatures-path': signatures_path } }
 = render 'projects/last_push'
 = render 'projects/files', commit: @last_commit, project: @project, ref: @ref, content_url: project_tree_path(@project, @id)

changelogs/unreleased/29121-rename-trace.yml

+---
+title: Replace wording trace with log
+merge_request:
+author:
+type: changed

changelogs/unreleased/expose-artifacts-to-merge-request-widget.yml

+---
+title: Expose arbitrary job artifacts in Merge Request widget
+merge_request: 18385
+author:
+type: added