Add latest changes from gitlab-org/gitlab@master

Gemfile

@@ -301,7 +301,7 @@ gem 'sentry-raven', '~> 2.9'
 gem 'premailer-rails', '~> 1.10.3'
 # LabKit: Tracing and Correlation
-gem 'gitlab-labkit', '~> 0.5'
+gem 'gitlab-labkit', '0.8.0'
 # I18n
 gem 'ruby_parser', '~> 3.8', require: false

Gemfile.lock

@@ -117,7 +117,7 @@ GEM
       activemodel (>= 5.0)
     brakeman (4.2.1)
     browser (2.5.3)
-    builder (3.2.3)
+    builder (3.2.4)
     bullet (6.0.2)
       activesupport (>= 3.0.0)
       uniform_notifier (~> 1.11)
@@ -362,7 +362,7 @@ GEM
     github-markup (1.7.0)
     gitlab-chronic (0.10.5)
       numerizer (~> 0.2)
-    gitlab-labkit (0.7.0)
+    gitlab-labkit (0.8.0)
       actionpack (>= 5.0.0, < 6.1.0)
       activesupport (>= 5.0.0, < 6.1.0)
       grpc (~> 1.19)
@@ -589,7 +589,7 @@ GEM
       activesupport (>= 4)
       railties (>= 4)
       request_store (~> 1.0)
-    loofah (2.3.1)
+    loofah (2.4.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.0.13)
@@ -1204,7 +1204,7 @@ DEPENDENCIES
   gitaly (~> 1.73.0)
   github-markup (~> 1.7.0)
   gitlab-chronic (~> 0.10.5)
-  gitlab-labkit (~> 0.5)
+  gitlab-labkit (= 0.8.0)
   gitlab-license (~> 1.0)
   gitlab-markup (~> 1.7.0)
   gitlab-net-dns (~> 0.9.1)

app/controllers/application_controller.rb

@@ -33,6 +33,7 @@ class ApplicationController < ActionController::Base
   before_action :check_impersonation_availability
   before_action :required_signup_info
+  around_action :set_current_context
   around_action :set_locale
   around_action :set_session_storage
@@ -448,6 +449,14 @@ class ApplicationController < ActionController::Base
     request.base_url
   end
+  def set_current_context(&block)
+    Gitlab::ApplicationContext.with_context(
+      user: -> { auth_user },
+      project: -> { @project },
+      namespace: -> { @group },
+      &block)
+  end
+
   def set_locale(&block)
     Gitlab::I18n.with_user_locale(current_user, &block)
   end

app/services/ci/find_exposed_artifacts_service.rb

@@ -46,6 +46,8 @@ module Ci
     # it could contain many. We only need to know whether it has 1 or more
     # artifacts, so fetching the first 2 would be sufficient.
     def first_2_metadata_entries_for_artifacts_paths(job)
+      return [] unless job.artifacts_metadata
+
       job.artifacts_paths
         .lazy
         .map { |path| job.artifacts_metadata_entry(path, recursive: true) }

changelogs/unreleased/194066-add-index-to-help-hashed-storage-migration-on-big-instances.yml

+---
+title: Add Index to help Hashed Storage migration on big instances
+merge_request: 22391
+author:
+type: performance

changelogs/unreleased/burndown-optimisations.yml

+---
+title: Performance improvements on milestone burndown chart
+merge_request: 22380
+author:
+type: performance

changelogs/unreleased/fix-no-artifacts-when-exposed.yml

+---
+title: Fix bug when trying to expose artifacts and no artifacts are produced by the job
+merge_request: 22378
+author:
+type: fixed

changelogs/unreleased/sh-fix-ci-lint-errors.yml

+---
+title: Gracefully error handle CI lint errors in artifacts section
+merge_request: 22388
+author:
+type: fixed

config/initializers/correlation_id.rb

-# frozen_string_literal: true
-
-Rails.application.config.middleware.use(Gitlab::Middleware::CorrelationId)

config/initializers/labkit_middleware.rb

+# frozen_string_literal: true
+
+Rails.application.config.middleware.use(Labkit::Middleware::Rack)

config/initializers/tracing.rb

@@ -2,7 +2,7 @@
 if Labkit::Tracing.enabled?
   Rails.application.configure do |config|
-    config.middleware.insert_after Gitlab::Middleware::CorrelationId, ::Labkit::Tracing::RackMiddleware
+    config.middleware.insert_after Labkit::Middleware::Rack, ::Labkit::Tracing::RackMiddleware
   end
   # Instrument the Sidekiq client

db/migrate/20200102170221_add_storage_version_index_to_projects.rb

+# frozen_string_literal: true
+
+class AddStorageVersionIndexToProjects < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, :id, where: 'storage_version < 2 or storage_version IS NULL', name: 'index_on_id_partial_with_legacy_storage'
+  end
+
+  def down
+    remove_concurrent_index :projects, :id, where: 'storage_version < 2 or storage_version IS NULL', name: 'index_on_id_partial_with_legacy_storage'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2019_12_29_140154) do
+ActiveRecord::Schema.define(version: 2020_01_02_170221) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -3349,6 +3349,7 @@ ActiveRecord::Schema.define(version: 2019_12_29_140154) do
     t.index ["creator_id"], name: "index_projects_on_creator_id"
     t.index ["description"], name: "index_projects_on_description_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["id", "repository_storage", "last_repository_updated_at"], name: "idx_projects_on_repository_storage_last_repository_updated_at"
+    t.index ["id"], name: "index_on_id_partial_with_legacy_storage", where: "((storage_version < 2) OR (storage_version IS NULL))"
     t.index ["id"], name: "index_projects_on_id_partial_for_visibility", unique: true, where: "(visibility_level = ANY (ARRAY[10, 20]))"
     t.index ["id"], name: "index_projects_on_mirror_and_mirror_trigger_builds_both_true", where: "((mirror IS TRUE) AND (mirror_trigger_builds IS TRUE))"
     t.index ["last_activity_at"], name: "index_projects_on_last_activity_at"

lib/api/api.rb

@@ -43,6 +43,14 @@ module API
       header['X-Content-Type-Options'] = 'nosniff'
     end
+    before do
+      Gitlab::ApplicationContext.push(
+        user: -> { current_user },
+        project: -> { @project },
+        namespace: -> { @group }
+      )
+    end
+
     # The locale is set to the current user's locale when `current_user` is loaded
     after { Gitlab::I18n.use_default_locale }

lib/api/commit_statuses.rb

@@ -71,27 +71,27 @@ module API
         ref = params[:ref]
         ref ||= pipeline&.ref
-        ref ||= @project.repository.branch_names_contains(commit.sha).first
+        ref ||= user_project.repository.branch_names_contains(commit.sha).first
         not_found! 'References for commit' unless ref
         name = params[:name] || params[:context] || 'default'
         unless pipeline
-          pipeline = @project.ci_pipelines.create!(
+          pipeline = user_project.ci_pipelines.create!(
             source: :external,
             sha: commit.sha,
             ref: ref,
             user: current_user,
-            protected: @project.protected_for?(ref))
+            protected: user_project.protected_for?(ref))
         end
         status = GenericCommitStatus.running_or_pending.find_or_initialize_by(
-          project: @project,
+          project: user_project,
           pipeline: pipeline,
           name: name,
           ref: ref,
           user: current_user,
-          protected: @project.protected_for?(ref)
+          protected: user_project.protected_for?(ref)
         )
         optional_attributes =
@@ -117,7 +117,7 @@ module API
             render_api_error!('invalid state', 400)
           end
-          MergeRequest.where(source_project: @project, source_branch: ref)
+          MergeRequest.where(source_project: user_project, source_branch: ref)
             .update_all(head_pipeline_id: pipeline.id) if pipeline.latest?
           present status, with: Entities::CommitStatus

lib/api/helpers/internal_helpers.rb

@@ -107,8 +107,10 @@ module API
         if params[:gl_repository]
           @project, @repo_type = Gitlab::GlRepository.parse(params[:gl_repository])
           @redirected_path = nil
-        else
+        elsif params[:project]
           @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse(params[:project])
+        else
+          @project, @repo_type, @redirected_path = nil, nil, nil
         end
       end
       # rubocop:enable Gitlab/ModuleWithInstanceVariables

lib/api/internal/base.rb

@@ -6,6 +6,13 @@ module API
     class Base < Grape::API
       before { authenticate_by_gitlab_shell_token! }
+      before do
+        Gitlab::ApplicationContext.push(
+          user: -> { actor&.user },
+          project: -> { project }
+        )
+      end
+
       helpers ::API::Helpers::InternalHelpers
       UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'.freeze
@@ -205,7 +212,12 @@ module API
           status 200
           response = Gitlab::InternalPostReceive::Response.new
+
+          # Try to load the project and users so we have the application context
+          # available for logging before we schedule any jobs.
           user = actor.user
+          project
+
           push_options = Gitlab::PushOptions.new(params[:push_options])
           response.reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease

lib/api/projects.rb

@@ -355,7 +355,7 @@ module API
       post ':id/unarchive' do
         authorize!(:archive_project, user_project)
-        ::Projects::UpdateService.new(@project, current_user, archived: false).execute
+        ::Projects::UpdateService.new(user_project, current_user, archived: false).execute
         present user_project, with: Entities::Project, current_user: current_user
       end

lib/gitlab/application_context.rb

+# frozen_string_literal: true
+
+module Gitlab
+  # A GitLab-rails specific accessor for `Labkit::Logging::ApplicationContext`
+  class ApplicationContext
+    include Gitlab::Utils::LazyAttributes
+
+    def self.with_context(args, &block)
+      application_context = new(**args)
+      Labkit::Context.with_context(application_context.to_lazy_hash, &block)
+    end
+
+    def self.push(args)
+      application_context = new(**args)
+      Labkit::Context.push(application_context.to_lazy_hash)
+    end
+
+    def initialize(user: nil, project: nil, namespace: nil)
+      @user, @project, @namespace = user, project, namespace
+    end
+
+    def to_lazy_hash
+      { user: -> { username },
+        project: -> { project_path },
+        root_namespace: -> { root_namespace_path } }
+    end
+
+    private
+
+    lazy_attr_reader :user, type: User
+    lazy_attr_reader :project, type: Project
+    lazy_attr_reader :namespace, type: Namespace
+
+    def project_path
+      project&.full_path
+    end
+
+    def username
+      user&.username
+    end
+
+    def root_namespace_path
+      if namespace
+        namespace.full_path_components.first
+      else
+        project&.full_path_components&.first
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/artifacts.rb

@@ -54,6 +54,11 @@ module Gitlab
           def expose_as_present?
             return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true)
+            # This duplicates the `validates :config, type: Hash` above,
+            # but Validatable currently doesn't halt the validation
+            # chain if it encounters a validation error.
+            return false unless @config.is_a?(Hash)
+
             !@config[:expose_as].nil?
           end
         end