Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/reports.gitlab-ci.yml

@@ -20,7 +20,7 @@ code_quality:
   variables:
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
-    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.6"
+    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.9"
   script:
     - |
       if ! docker info &>/dev/null; then

app/assets/javascripts/behaviors/markdown/render_mermaid.js

@@ -35,6 +35,7 @@ function renderMermaids($els) {
         // mermaidAPI options
         theme: 'neutral',
         flowchart: {
+          useMaxWidth: true,
           htmlLabels: false,
         },
         securityLevel: 'strict',

app/finders/projects/export_job_finder.rb

+# frozen_string_literal: true
+
+module Projects
+  class ExportJobFinder
+    InvalidExportJobStatusError = Class.new(StandardError)
+    attr_reader :project, :params
+
+    def initialize(project, params = {})
+      @project = project
+      @params = params
+    end
+
+    def execute
+      export_jobs = project.export_jobs
+      export_jobs = by_status(export_jobs)
+
+      export_jobs
+    end
+
+    private
+
+    def by_status(export_jobs)
+      return export_jobs unless params[:status]
+      raise InvalidExportJobStatusError, 'Invalid export job status' unless ProjectExportJob.state_machines[:status].states.map(&:name).include?(params[:status])
+
+      export_jobs.with_status(params[:status])
+    end
+  end
+end

app/models/ci/bridge.rb

@@ -10,6 +10,7 @@ module Ci
     include HasRef
     InvalidBridgeTypeError = Class.new(StandardError)
+    InvalidTransitionError = Class.new(StandardError)
     belongs_to :project
     belongs_to :trigger_request

app/models/project.rb

@@ -186,6 +186,7 @@ class Project < ApplicationRecord
   has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
   has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+  has_many :export_jobs, class_name: 'ProjectExportJob'
   has_one :project_repository, inverse_of: :project
   has_one :incident_management_setting, inverse_of: :project, class_name: 'IncidentManagement::ProjectIncidentManagementSetting'
   has_one :error_tracking_setting, inverse_of: :project, class_name: 'ErrorTracking::ProjectErrorTrackingSetting'
@@ -1850,10 +1851,12 @@ class Project < ApplicationRecord
   end
   def export_status
-    if export_in_progress?
+    if regeneration_in_progress?
+      :regeneration_in_progress
+    elsif export_enqueued?
+      :queued
+    elsif export_in_progress?
       :started
-    elsif after_export_in_progress?
-      :after_export_action
     elsif export_file_exists?
       :finished
     else
@@ -1862,11 +1865,19 @@ class Project < ApplicationRecord
   end
   def export_in_progress?
-    import_export_shared.active_export_count > 0
+    strong_memoize(:export_in_progress) do
+      ::Projects::ExportJobFinder.new(self, { status: :started }).execute.present?
+    end
+  end
+
+  def export_enqueued?
+    strong_memoize(:export_enqueued) do
+      ::Projects::ExportJobFinder.new(self, { status: :queued }).execute.present?
+    end
   end
-  def after_export_in_progress?
-    import_export_shared.after_export_in_progress?
+  def regeneration_in_progress?
+    (export_enqueued? || export_in_progress?) && export_file_exists?
   end
   def remove_exports

app/models/project_export_job.rb

+# frozen_string_literal: true
+
+class ProjectExportJob < ApplicationRecord
+  belongs_to :project
+
+  validates :project, :jid, :status, presence: true
+
+  state_machine :status, initial: :queued do
+    event :start do
+      transition [:queued] => :started
+    end
+
+    event :finish do
+      transition [:started] => :finished
+    end
+
+    event :fail_op do
+      transition [:queued, :started] => :failed
+    end
+
+    state :queued, value: 0
+    state :started, value: 1
+    state :finished, value: 2
+    state :failed, value: 3
+  end
+end

app/services/ci/create_cross_project_pipeline_service.rb

@@ -52,6 +52,11 @@ module Ci
           subject.drop!(:downstream_pipeline_creation_failed)
         end
       end
+    rescue StateMachines::InvalidTransition => e
+      Gitlab::ErrorTracking.track_exception(
+        Ci::Bridge::InvalidTransitionError.new(e.message),
+        bridge_id: bridge.id,
+        downstream_pipeline_id: pipeline.id)
     end
     def ensure_preconditions!(target_ref)

app/services/ci/pipeline_bridge_status_service.rb

@@ -5,7 +5,14 @@ module Ci
     def execute(pipeline)
       return unless pipeline.bridge_triggered?
-      pipeline.source_bridge.inherit_status_from_downstream!(pipeline)
+      begin
+        pipeline.source_bridge.inherit_status_from_downstream!(pipeline)
+      rescue StateMachines::InvalidTransition => e
+        Gitlab::ErrorTracking.track_exception(
+          Ci::Bridge::InvalidTransitionError.new(e.message),
+          bridge_id: pipeline.source_bridge.id,
+          downstream_pipeline_id: pipeline.id)
+      end
     end
   end
 end

app/workers/all_queues.yml

@@ -234,6 +234,13 @@
   :resource_boundary: :cpu
   :weight: 1
   :idempotent: 
+- :name: cronjob:stuck_export_jobs
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :default
+  :resource_boundary: :cpu
+  :weight: 1
+  :idempotent: 
 - :name: cronjob:stuck_import_jobs
   :feature_category: :importers
   :has_external_dependencies: 

app/workers/concerns/project_export_options.rb

+# frozen_string_literal: true
+
+module ProjectExportOptions
+  extend ActiveSupport::Concern
+
+  EXPORT_RETRY_COUNT = 3
+
+  included do
+    sidekiq_options retry: EXPORT_RETRY_COUNT, status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION
+
+    # We mark the project export as failed once we have exhausted all retries
+    sidekiq_retries_exhausted do |job|
+      project = Project.find(job['args'][1])
+      # rubocop: disable CodeReuse/ActiveRecord
+      job = project.export_jobs.find_by(jid: job["jid"])
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      if job&.fail_op
+        Sidekiq.logger.info "Job #{job['jid']} for project #{project.id} has been set to failed state"
+      else
+        Sidekiq.logger.error "Failed to set Job #{job['jid']} for project #{project.id} to failed state"
+      end
+    end
+  end
+end

app/workers/project_export_worker.rb

@@ -3,17 +3,24 @@
 class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker
   include ApplicationWorker
   include ExceptionBacktrace
-  sidekiq_options retry: 3
+  include ProjectExportOptions
   feature_category :importers
   worker_resource_boundary :memory
   def perform(current_user_id, project_id, after_export_strategy = {}, params = {})
     current_user = User.find(current_user_id)
     project = Project.find(project_id)
+    export_job = project.export_jobs.safe_find_or_create_by(jid: self.jid)
     after_export = build!(after_export_strategy)
+    export_job&.start
+
     ::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export)
+
+    export_job&.finish
+  rescue ActiveRecord::RecordNotFound, Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError => e
+    logger.error("Failed to export project #{project_id}: #{e.message}")
   end
   private

app/workers/stuck_export_jobs_worker.rb

+# frozen_string_literal: true
+
+# rubocop:disable Scalability/IdempotentWorker
+class StuckExportJobsWorker
+  include ApplicationWorker
+  # rubocop:disable Scalability/CronWorkerContext
+  # This worker updates export states inline and does not schedule
+  # other jobs.
+  include CronjobQueue
+  # rubocop:enable Scalability/CronWorkerContext
+
+  feature_category :importers
+  worker_resource_boundary :cpu
+
+  EXPORT_JOBS_EXPIRATION = 6.hours.to_i
+
+  def perform
+    failed_jobs_count = mark_stuck_jobs_as_failed!
+
+    Gitlab::Metrics.add_event(:stuck_export_jobs,
+                              failed_jobs_count: failed_jobs_count)
+  end
+
+  private
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  def mark_stuck_jobs_as_failed!
+    jids_and_ids = enqueued_exports.pluck(:jid, :id).to_h
+
+    completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
+    return unless completed_jids.any?
+
+    completed_ids = jids_and_ids.values_at(*completed_jids)
+
+    # We select the export states again, because they may have transitioned from
+    # started to finished while we were looking up their Sidekiq status.
+    completed_jobs = enqueued_exports.where(id: completed_ids)
+
+    Sidekiq.logger.info(
+      message: 'Marked stuck export jobs as failed',
+      job_ids: completed_jobs.map(&:jid)
+    )
+
+    completed_jobs.each do |job|
+      job.fail_op
+    end.count
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  def enqueued_exports
+    ProjectExportJob.with_status([:started, :queued])
+  end
+end
+# rubocop:enable Scalability/IdempotentWorker

changelogs/unreleased/204774-quick-actions-executed-in-multiline-inline-code.yml

+---
+title: Fix quick actions executing in multiline inline code when placed on its own line
+merge_request: 24933
+author: Pavlo Dudchenko
+type: fixed

changelogs/unreleased/26712-Update-GitLab-codeclimate-to-head.yml

+---
+title: Update GitLab's codeclimate to 0.85.9
+merge_request: 26712
+author: Eddie Stubbington
+type: other

changelogs/unreleased/fix-export-state-logic.yml

+---
+title: Fix logic to determine project export state and add regeneration_in_progress state
+merge_request: 23664
+author:
+type: fixed

changelogs/unreleased/fix-mermaid-flow-chart-width.yml

+---
+title: Fix Mermaid flowchart width
+merge_request: 26848
+author: julien MILLAU
+type: fixed

config/initializers/1_settings.rb

@@ -453,6 +453,9 @@ Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'Rem
 Settings.cron_jobs['stuck_import_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_import_jobs_worker']['cron'] ||= '15 * * * *'
 Settings.cron_jobs['stuck_import_jobs_worker']['job_class'] = 'StuckImportJobsWorker'
+Settings.cron_jobs['stuck_export_jobs_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['stuck_export_jobs_worker']['cron'] ||= '30 * * * *'
+Settings.cron_jobs['stuck_export_jobs_worker']['job_class'] = 'StuckExportJobsWorker'
 Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= nil # This is dynamically loaded in the sidekiq initializer
 Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'

db/migrate/20200311165635_create_project_export_jobs.rb

+# frozen_string_literal: true
+
+class CreateProjectExportJobs < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    create_table :project_export_jobs do |t|
+      t.references :project, index: false, null: false, foreign_key: { on_delete: :cascade }
+      t.timestamps_with_timezone null: false
+      t.integer :status, limit: 2, null: false, default: 0
+      t.string :jid, limit: 100, null: false, unique: true
+
+      t.index [:project_id, :jid]
+      t.index [:jid], unique: true
+      t.index [:status]
+      t.index [:project_id, :status]
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2020_03_10_135823) do
+ActiveRecord::Schema.define(version: 2020_03_11_165635) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -3242,6 +3242,18 @@ ActiveRecord::Schema.define(version: 2020_03_10_135823) do
     t.string "organization_name"
   end
+  create_table "project_export_jobs", force: :cascade do |t|
+    t.bigint "project_id", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "status", limit: 2, default: 0, null: false
+    t.string "jid", limit: 100, null: false
+    t.index ["jid"], name: "index_project_export_jobs_on_jid", unique: true
+    t.index ["project_id", "jid"], name: "index_project_export_jobs_on_project_id_and_jid"
+    t.index ["project_id", "status"], name: "index_project_export_jobs_on_project_id_and_status"
+    t.index ["status"], name: "index_project_export_jobs_on_status"
+  end
+
   create_table "project_feature_usages", primary_key: "project_id", id: :integer, default: nil, force: :cascade do |t|
     t.datetime "jira_dvcs_cloud_last_sync_at"
     t.datetime "jira_dvcs_server_last_sync_at"
@@ -5017,6 +5029,7 @@ ActiveRecord::Schema.define(version: 2020_03_10_135823) do
   add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade
   add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade
   add_foreign_key "project_error_tracking_settings", "projects", on_delete: :cascade
+  add_foreign_key "project_export_jobs", "projects", on_delete: :cascade
   add_foreign_key "project_feature_usages", "projects", on_delete: :cascade
   add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
   add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade

doc/administration/integration/plantuml.md

@@ -122,12 +122,12 @@ our AsciiDoc snippets, wikis and repos using delimited blocks:
 - **Markdown**
-  ~~~markdown
+  ````markdown
   ```plantuml
   Bob -> Alice : hello
   Alice -> Bob : hi
   ```
-  ~~~
+  ````
 - **AsciiDoc**