Add latest changes from gitlab-org/gitlab@master

app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue

@@ -115,7 +115,10 @@ export default {
               data-qa-selector="merge_request_pipeline_info_content"
             >
               {{ pipeline.details.name }}
-              <gl-link :href="pipeline.path" class="pipeline-id font-weight-normal pipeline-number"
+              <gl-link
+                :href="pipeline.path"
+                class="pipeline-id font-weight-normal pipeline-number"
+                data-qa-selector="pipeline_link"
                 >#{{ pipeline.id }}</gl-link
               >
               {{ pipeline.details.status.label }}

app/services/git/base_hooks_service.rb

@@ -15,8 +15,6 @@ module Git
       # Not a hook, but it needs access to the list of changed commits
       enqueue_invalidate_cache
-      update_remote_mirrors
-
       success
     end
@@ -121,13 +119,6 @@ module Git
       {}
     end
-    def update_remote_mirrors
-      return unless project.has_remote_mirror?
-
-      project.mark_stuck_remote_mirrors_as_failed!
-      project.update_remote_mirrors
-    end
-
     def log_pipeline_errors(exception)
       data = {
         class: self.class.name,

app/services/git/branch_push_service.rb

@@ -57,13 +57,6 @@ module Git
       Ci::StopEnvironmentsService.new(project, current_user).execute(branch_name)
     end
-    def update_remote_mirrors
-      return unless project.has_remote_mirror?
-
-      project.mark_stuck_remote_mirrors_as_failed!
-      project.update_remote_mirrors
-    end
-
     def execute_related_hooks
       BranchHooksService.new(project, current_user, params).execute
     end

app/services/system_note_service.rb

@@ -16,20 +16,9 @@ module SystemNoteService
   # existing_commits - Array of Commits added in a previous push
   # oldrev           - Optional String SHA of a previous Commit
   #
-  # See new_commit_summary and existing_commit_summary.
-  #
   # Returns the created Note object
   def add_commits(noteable, project, author, new_commits, existing_commits = [], oldrev = nil)
-    total_count  = new_commits.length + existing_commits.length
-    commits_text = "#{total_count} commit".pluralize(total_count)
-
-    text_parts = ["added #{commits_text}"]
-    text_parts << commits_list(noteable, new_commits, existing_commits, oldrev)
-    text_parts << "[Compare with previous version](#{diff_comparison_path(noteable, project, oldrev)})"
-
-    body = text_parts.join("\n\n")
-
-    create_note(NoteSummary.new(noteable, project, author, body, action: 'commit', commit_count: total_count))
+    ::SystemNotes::CommitService.new(noteable: noteable, project: project, author: author).add_commits(new_commits, existing_commits, oldrev)
   end
   # Called when a commit was tagged
@@ -41,10 +30,7 @@ module SystemNoteService
   #
   # Returns the created Note object
   def tag_commit(noteable, project, author, tag_name)
-    link = url_helpers.project_tag_path(project, id: tag_name)
-    body = "tagged commit #{noteable.sha} to [`#{tag_name}`](#{link})"
-
-    create_note(NoteSummary.new(noteable, project, author, body, action: 'tag'))
+    ::SystemNotes::CommitService.new(noteable: noteable, project: project, author: author).tag_commit(tag_name)
   end
   # Called when the assignee of a Noteable is changed or removed
@@ -497,17 +483,6 @@ module SystemNoteService
     notes_for_mentioner(mentioner, noteable, notes).exists?
   end
-  # Build an Array of lines detailing each commit added in a merge request
-  #
-  # new_commits - Array of new Commit objects
-  #
-  # Returns an Array of Strings
-  def new_commit_summary(new_commits)
-    new_commits.collect do |commit|
-      content_tag('li', "#{commit.short_id} - #{commit.title}")
-    end
-  end
-
   # Called when the status of a Task has changed
   #
   # noteable  - Noteable object.
@@ -637,71 +612,10 @@ module SystemNoteService
     "#{cross_reference_note_prefix}#{gfm_reference}"
   end
-  # Builds a list of existing and new commits according to existing_commits and
-  # new_commits methods.
-  # Returns a String wrapped in `ul` and `li` tags.
-  def commits_list(noteable, new_commits, existing_commits, oldrev)
-    existing_commit_summary = existing_commit_summary(noteable, existing_commits, oldrev)
-    new_commit_summary = new_commit_summary(new_commits).join
-
-    content_tag('ul', "#{existing_commit_summary}#{new_commit_summary}".html_safe)
-  end
-
-  # Build a single line summarizing existing commits being added in a merge
-  # request
-  #
-  # noteable         - MergeRequest object
-  # existing_commits - Array of existing Commit objects
-  # oldrev           - Optional String SHA of a previous Commit
-  #
-  # Examples:
-  #
-  #   "* ea0f8418...2f4426b7 - 24 commits from branch `master`"
-  #
-  #   "* ea0f8418..4188f0ea - 15 commits from branch `fork:master`"
-  #
-  #   "* ea0f8418 - 1 commit from branch `feature`"
-  #
-  # Returns a newline-terminated String
-  def existing_commit_summary(noteable, existing_commits, oldrev = nil)
-    return '' if existing_commits.empty?
-
-    count = existing_commits.size
-
-    commit_ids = if count == 1
-                   existing_commits.first.short_id
-                 else
-                   if oldrev && !Gitlab::Git.blank_ref?(oldrev)
-                     "#{Commit.truncate_sha(oldrev)}...#{existing_commits.last.short_id}"
-                   else
-                     "#{existing_commits.first.short_id}..#{existing_commits.last.short_id}"
-                   end
-                 end
-
-    commits_text = "#{count} commit".pluralize(count)
-
-    branch = noteable.target_branch
-    branch = "#{noteable.target_project_namespace}:#{branch}" if noteable.for_fork?
-
-    branch_name = content_tag('code', branch)
-    content_tag('li', "#{commit_ids} - #{commits_text} from branch #{branch_name}".html_safe)
-  end
-
   def url_helpers
     @url_helpers ||= Gitlab::Routing.url_helpers
   end
-  def diff_comparison_path(merge_request, project, oldrev)
-    diff_id = merge_request.merge_request_diff.id
-
-    url_helpers.diffs_project_merge_request_path(
-      project,
-      merge_request,
-      diff_id: diff_id,
-      start_sha: oldrev
-    )
-  end
-
   def content_tag(*args)
     ActionController::Base.helpers.content_tag(*args)
   end

app/services/system_notes/base_service.rb

+# frozen_string_literal: true
+
+module SystemNotes
+  class BaseService
+    attr_reader :noteable, :project, :author
+
+    def initialize(noteable: nil, author: nil, project: nil)
+      @noteable = noteable
+      @project = project
+      @author = author
+    end
+
+    protected
+
+    def create_note(note_summary)
+      note = Note.create(note_summary.note.merge(system: true))
+      note.system_note_metadata = SystemNoteMetadata.new(note_summary.metadata) if note_summary.metadata?
+
+      note
+    end
+
+    def content_tag(*args)
+      ActionController::Base.helpers.content_tag(*args)
+    end
+
+    def url_helpers
+      @url_helpers ||= Gitlab::Routing.url_helpers
+    end
+  end
+end

app/services/system_notes/commit_service.rb

+# frozen_string_literal: true
+
+module SystemNotes
+  class CommitService < ::SystemNotes::BaseService
+    # Called when commits are added to a Merge Request
+    #
+    # new_commits      - Array of Commits added since last push
+    # existing_commits - Array of Commits added in a previous push
+    # oldrev           - Optional String SHA of a previous Commit
+    #
+    # See new_commit_summary and existing_commit_summary.
+    #
+    # Returns the created Note object
+    def add_commits(new_commits, existing_commits = [], oldrev = nil)
+      total_count  = new_commits.length + existing_commits.length
+      commits_text = "#{total_count} commit".pluralize(total_count)
+
+      text_parts = ["added #{commits_text}"]
+      text_parts << commits_list(noteable, new_commits, existing_commits, oldrev)
+      text_parts << "[Compare with previous version](#{diff_comparison_path(noteable, project, oldrev)})"
+
+      body = text_parts.join("\n\n")
+
+      create_note(NoteSummary.new(noteable, project, author, body, action: 'commit', commit_count: total_count))
+    end
+
+    # Called when a commit was tagged
+    #
+    # tag_name  - The created tag name
+    #
+    # Returns the created Note object
+    def tag_commit(tag_name)
+      link = url_helpers.project_tag_path(project, id: tag_name)
+      body = "tagged commit #{noteable.sha} to [`#{tag_name}`](#{link})"
+
+      create_note(NoteSummary.new(noteable, project, author, body, action: 'tag'))
+    end
+
+    # Build an Array of lines detailing each commit added in a merge request
+    #
+    # new_commits - Array of new Commit objects
+    #
+    # Returns an Array of Strings
+    def new_commit_summary(new_commits)
+      new_commits.collect do |commit|
+        content_tag('li', "#{commit.short_id} - #{commit.title}")
+      end
+    end
+
+    private
+
+    # Builds a list of existing and new commits according to existing_commits and
+    # new_commits methods.
+    # Returns a String wrapped in `ul` and `li` tags.
+    def commits_list(noteable, new_commits, existing_commits, oldrev)
+      existing_commit_summary = existing_commit_summary(noteable, existing_commits, oldrev)
+      new_commit_summary = new_commit_summary(new_commits).join
+
+      content_tag('ul', "#{existing_commit_summary}#{new_commit_summary}".html_safe)
+    end
+
+    # Build a single line summarizing existing commits being added in a merge
+    # request
+    #
+    # existing_commits - Array of existing Commit objects
+    # oldrev           - Optional String SHA of a previous Commit
+    #
+    # Examples:
+    #
+    #   "* ea0f8418...2f4426b7 - 24 commits from branch `master`"
+    #
+    #   "* ea0f8418..4188f0ea - 15 commits from branch `fork:master`"
+    #
+    #   "* ea0f8418 - 1 commit from branch `feature`"
+    #
+    # Returns a newline-terminated String
+    def existing_commit_summary(noteable, existing_commits, oldrev = nil)
+      return '' if existing_commits.empty?
+
+      count = existing_commits.size
+
+      commit_ids = if count == 1
+                     existing_commits.first.short_id
+                   else
+                     if oldrev && !Gitlab::Git.blank_ref?(oldrev)
+                       "#{Commit.truncate_sha(oldrev)}...#{existing_commits.last.short_id}"
+                     else
+                       "#{existing_commits.first.short_id}..#{existing_commits.last.short_id}"
+                     end
+                   end
+
+      commits_text = "#{count} commit".pluralize(count)
+
+      branch = noteable.target_branch
+      branch = "#{noteable.target_project_namespace}:#{branch}" if noteable.for_fork?
+
+      branch_name = content_tag('code', branch)
+      content_tag('li', "#{commit_ids} - #{commits_text} from branch #{branch_name}".html_safe)
+    end
+
+    def diff_comparison_path(merge_request, project, oldrev)
+      diff_id = merge_request.merge_request_diff.id
+
+      url_helpers.diffs_project_merge_request_path(
+        project,
+        merge_request,
+        diff_id: diff_id,
+        start_sha: oldrev
+      )
+    end
+  end
+end

app/workers/post_receive.rb

@@ -70,6 +70,7 @@ class PostReceive
       refs << ref
     end
+    update_remote_mirrors(post_received)
     after_project_changes_hooks(post_received, user, refs.to_a, changes)
   end
@@ -93,6 +94,16 @@ class PostReceive
     )
   end
+  def update_remote_mirrors(post_received)
+    return unless post_received.includes_branches? || post_received.includes_tags?
+
+    project = post_received.project
+    return unless project.has_remote_mirror?
+
+    project.mark_stuck_remote_mirrors_as_failed!
+    project.update_remote_mirrors
+  end
+
   def after_project_changes_hooks(post_received, user, refs, changes)
     hook_data = Gitlab::DataBuilder::Repository.update(post_received.project, user, changes, refs)
     SystemHooksService.new.execute_hooks(hook_data, :repository_update_hooks)

changelogs/unreleased/bvl-only-one-pushmirror-per-push.yml

+---
+title: Only schedule updating push-mirrors once per push
+merge_request: 17902
+author:
+type: performance

changelogs/unreleased/sh-enable-google-api-retries.yml

+---
+title: Enable Google API retries for uploads
+merge_request: 18040
+author:
+type: fixed

config/initializers/google_api_client.rb

@@ -10,6 +10,12 @@
 #
 require 'google/apis/container_v1beta1'
+require 'google/apis/options'
+
+# As stated in https://github.com/googleapis/google-api-ruby-client#errors--retries,
+# enabling retries is strongly encouraged but disabled by default. Large uploads
+# that may hit timeouts will mainly benefit from this.
+Google::Apis::RequestOptions.default.retries = 3 if Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_GOOGLE_API_RETRIES', true))
 Google::Apis::ContainerV1beta1::AddonsConfig::Representation.tap do |representation|
   representation.hash :cloud_run_config, as: 'cloudRunConfig'

doc/administration/high_availability/monitoring_node.md

@@ -34,6 +34,9 @@ Omnibus:
    prometheus['listen_address'] = '0.0.0.0:9090'
    prometheus['monitor_kubernetes'] = false
+   # Enable Login form
+   grafana['disable_login_form'] = false
+
    # Enable Grafana
    grafana['enable'] = true
    grafana['admin_password'] = 'toomanysecrets'
@@ -63,6 +66,7 @@ Omnibus:
    sidekiq['enable'] = false
    unicorn['enable'] = false
    node_exporter['enable'] = false
+   gitlab_exporter['enable'] = false
    ```
 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration.

doc/user/permissions.md

@@ -48,7 +48,7 @@ The following table depicts the various user permission levels in a project.
 | View Insights charts **(ULTIMATE)**               | ✓       | ✓          | ✓           | ✓        | ✓      |
 | View approved/blacklisted licenses **(ULTIMATE)** | ✓       | ✓          | ✓           | ✓        | ✓      |
 | View License Compliance reports **(ULTIMATE)**    | ✓ (*1*) | ✓          | ✓           | ✓        | ✓      |
-| View Security reports **(ULTIMATE)**              | ✓ (*1*) | ✓          | ✓           | ✓        | ✓      |
+| View Security reports **(ULTIMATE)**              | ✓ (*3*) | ✓          | ✓           | ✓        | ✓      |
 | View Dependency list **(ULTIMATE)**               | ✓ (*1*) | ✓          | ✓           | ✓        | ✓      |
 | View licenses in Dependency list **(ULTIMATE)**   | ✓ (*1*) | ✓          | ✓           | ✓        | ✓      |
 | View [Design Management](project/issues/design_management.md) pages **(PREMIUM)** | ✓ (*1*) | ✓          | ✓           | ✓        | ✓      |

lib/gitlab/import_export/fast_hash_serializer.rb

@@ -26,6 +26,51 @@ module Gitlab
     class FastHashSerializer
       attr_reader :subject, :tree
+      # Usage of this class results in delayed
+      # serialization of relation. The serialization
+      # will be triggered when the `JSON.generate`
+      # is exected.
+      #
+      # This class uses memory-optimised, lazily
+      # initialised, fast to recycle relation
+      # serialization.
+      #
+      # The `JSON.generate` does use `#to_json`,
+      # that returns raw JSON content that is written
+      # directly to file.
+      class JSONBatchRelation
+        include Gitlab::Utils::StrongMemoize
+
+        def initialize(relation, options, preloads)
+          @relation = relation
+          @options = options
+          @preloads = preloads
+        end
+
+        def raw_json
+          strong_memoize(:raw_json) do
+            result = +''
+
+            batch = @relation
+            batch = batch.preload(@preloads) if @preloads
+            batch.each do |item|
+              result.concat(",") unless result.empty?
+              result.concat(item.to_json(@options))
+            end
+
+            result
+          end
+        end
+
+        def to_json(options = {})
+          raw_json
+        end
+
+        def as_json(*)
+          raise NotImplementedError
+        end
+      end
+
       BATCH_SIZE = 100
       def initialize(subject, tree, batch_size: BATCH_SIZE)
@@ -34,8 +79,11 @@ module Gitlab
         @tree = tree
       end
-      # Serializes the subject into a Hash for the given option tree
-      # (e.g. Project#as_json)
+      # With the usage of `JSONBatchRelation`, it returns partially
+      # serialized hash which is not easily accessible.
+      # It means you can only manipulate and replace top-level objects.
+      # All future mutations of the hash (such as `fix_project_tree`)
+      # should be aware of that.
       def execute
         simple_serialize.merge(serialize_includes)
       end
@@ -85,13 +133,20 @@ module Gitlab
           return record.as_json(options)
         end
-        # has-many relation
         data = []
-        record.in_batches(of: @batch_size) do |batch| # rubocop:disable Cop/InBatches
-          batch = batch.preload(preloads[key]) if preloads&.key?(key)
-          data += batch.as_json(options)
+        # rubocop:disable Cop/InBatches
+        # If we put `rubocop:disable` inline after `do |batch|`,
+        # `Cop/LineBreakAroundConditionalBlock` will fail
+        record.in_batches(of: @batch_size) do |batch|
+          if Feature.enabled?(:export_fast_serialize_with_raw_json, default_enabled: true)
+            data.append(JSONBatchRelation.new(batch, options, preloads[key]).tap(&:raw_json))
+          else
+            batch = batch.preload(preloads[key]) if preloads&.key?(key)
+            data += batch.as_json(options)
+          end
         end
+        # rubocop:enable Cop/InBatches
         data
       end

lib/gitlab/import_export/project_tree_saver.rb

@@ -20,7 +20,8 @@ module Gitlab
         project_tree = serialize_project_tree
         fix_project_tree(project_tree)
-        File.write(full_path, project_tree.to_json)
+        project_tree_json = JSON.generate(project_tree)
+        File.write(full_path, project_tree_json)
         true
       rescue => e
@@ -30,6 +31,8 @@ module Gitlab
       private
+      # Aware that the resulting hash needs to be pure-hash and
+      # does not include any AR objects anymore, only objects that run `.to_json`
       def fix_project_tree(project_tree)
         if @params[:description].present?
           project_tree['description'] = @params[:description]

qa/qa.rb

@@ -357,6 +357,7 @@ module QA
     # Classes describing components that are used by several pages.
     #
     module Component
+      autoload :CiBadgeLink, 'qa/page/component/ci_badge_link'
       autoload :ClonePanel, 'qa/page/component/clone_panel'
       autoload :LazyLoader, 'qa/page/component/lazy_loader'
       autoload :LegacyClonePanel, 'qa/page/component/legacy_clone_panel'

qa/qa/page/base.rb

@@ -95,7 +95,7 @@ module QA
       # replace with (..., page = self.class)
       def click_element(name, page = nil, text: nil)
-        find_element(name, text: nil).click
+        find_element(name, text: text).click
         page.validate_elements_present! if page
       end

qa/qa/page/component/ci_badge_link.rb

+# frozen_string_literal: true
+
+module QA
+  module Page
+    module Component
+      module CiBadgeLink
+        COMPLETED_STATUSES = %w[passed failed canceled blocked skipped manual].freeze # excludes created, pending, running
+        PASSED_STATUS = 'passed'.freeze
+
+        def self.included(base)
+          base.view 'app/assets/javascripts/vue_shared/components/ci_badge_link.vue' do
+            element :status_badge
+          end
+        end
+
+        def status_badge
+          find_element(:status_badge).text
+        end
+
+        def successful?(timeout: 60)
+          raise "Timed out waiting for the status to be a valid completed state" unless completed?(timeout: timeout)
+
+          status_badge == PASSED_STATUS
+        end
+
+        private
+
+        def completed?(timeout: 60)
+          wait(reload: false, max: timeout) do
+            COMPLETED_STATUSES.include?(status_badge)
+          end
+        end
+      end
+    end
+  end
+end

qa/qa/page/merge_request/show.rb

@@ -14,6 +14,7 @@ module QA
         view 'app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue' do
           element :merge_request_pipeline_info_content
+          element :pipeline_link
         end
         view 'app/assets/javascripts/vue_merge_request_widget/components/states/ready_to_merge.vue' do
@@ -59,6 +60,18 @@ module QA
           element :edit_button
         end
+        def click_discussions_tab
+          click_element :notes_tab
+        end
+
+        def click_diffs_tab
+          click_element :diffs_tab
+        end
+
+        def click_pipeline_link
+          click_element :pipeline_link
+        end
+
         def fast_forward_possible?
           has_no_text?('Fast-forward merge is not possible')
         end
@@ -156,14 +169,6 @@ module QA
           click_element :squash_checkbox
         end
-        def click_discussions_tab
-          click_element :notes_tab
-        end
-
-        def click_diffs_tab
-          click_element :diffs_tab
-        end
-
         def add_comment_to_diff(text)
           wait(interval: 5) do
             has_text?("No newline at end of file")

qa/qa/page/project/job/show.rb

@@ -3,17 +3,12 @@
 module QA::Page
   module Project::Job
     class Show < QA::Page::Base
-      COMPLETED_STATUSES = %w[passed failed canceled blocked skipped manual].freeze # excludes created, pending, running
-      PASSED_STATUS = 'passed'.freeze
+      include Component::CiBadgeLink
       view 'app/assets/javascripts/jobs/components/job_log.vue' do
         element :build_trace
       end
-      view 'app/assets/javascripts/vue_shared/components/ci_badge_link.vue' do
-        element :status_badge
-      end
-
       view 'app/assets/javascripts/jobs/components/stages_dropdown.vue' do
         element :pipeline_path
       end
@@ -45,16 +40,6 @@ module QA::Page
           has_element?(:build_trace, wait: 1)
         end
       end
-
-      def completed?(timeout: 60)
-        wait(reload: false, max: timeout) do
-          COMPLETED_STATUSES.include?(status_badge)
-        end
-      end
-
-      def status_badge
-        find_element(:status_badge).text
-      end
     end
   end
 end

qa/qa/page/project/pipeline/show.rb

@@ -3,6 +3,8 @@
 module QA::Page
   module Project::Pipeline
     class Show < QA::Page::Base
+      include Component::CiBadgeLink
+
       view 'app/assets/javascripts/vue_shared/components/header_ci_component.vue' do
         element :pipeline_header, /header class.*ci-header-container.*/ # rubocop:disable QA/ElementWithPattern
       end
@@ -38,6 +40,14 @@ module QA::Page
         end
       end
+      def has_job?(job_name)
+        has_element?(:job_link, text: job_name)
+      end
+
+      def has_no_job?(job_name)
+        has_no_element?(:job_link, text: job_name)
+      end
+
       def has_tag?(tag_name)
         within_element(:pipeline_badges) do
           has_selector?('.badge', text: tag_name)
@@ -45,7 +55,11 @@ module QA::Page
       end
       def click_job(job_name)
-        find_element(:job_link, text: job_name).click
+        click_element(:job_link, text: job_name)
+      end
+
+      def click_linked_job(project_name)
+        click_element(:linked_pipeline_button, text: /#{project_name}/)
       end
       def click_on_first_job