Add latest changes from gitlab-org/gitlab@master

app/workers/stuck_ci_jobs_worker.rb

@@ -2,7 +2,7 @@
 class StuckCiJobsWorker
   include ApplicationWorker
-  include CronjobQueue
+  include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
   feature_category :continuous_integration
   worker_resource_boundary :cpu

app/workers/stuck_import_jobs_worker.rb

@@ -2,7 +2,7 @@
 class StuckImportJobsWorker
   include ApplicationWorker
-  include CronjobQueue
+  include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
   feature_category :importers
   worker_resource_boundary :cpu

app/workers/stuck_merge_jobs_worker.rb

@@ -2,7 +2,7 @@
 class StuckMergeJobsWorker
   include ApplicationWorker
-  include CronjobQueue
+  include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
   feature_category :source_code_management

app/workers/trending_projects_worker.rb

@@ -2,7 +2,7 @@
 class TrendingProjectsWorker
   include ApplicationWorker
-  include CronjobQueue
+  include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
   feature_category :source_code_management

changelogs/unreleased/14624-vendored-template-for-license-scanning.yml

+---
+title: Introduce license_scanning CI template
+merge_request: 22773
+author:
+type: added

changelogs/unreleased/197371-show-notification-on-related-error-close.yml

+---
+title: Close related GitLab issue on Sentry error resolve
+merge_request: 23610
+author:
+type: added

changelogs/unreleased/197926-initial-package-detail-title.yml

+---
+title: Updated package details page header to begin updating the page design.
+merge_request: 24055
+author:
+type: added

changelogs/unreleased/34262-readonly-ldap-attributes.yml

+---
+title: Make name, email, and location attributes readonly for LDAP enabled instances
+merge_request: 24049
+author:
+type: changed

db/migrate/20200203025400_default_lock_version_to_zero_for_merge_requests.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class DefaultLockVersionToZeroForMergeRequests < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    with_lock_retries do
+      change_column_default :merge_requests, :lock_version, from: nil, to: 0
+    end
+  end
+end

db/migrate/20200203025602_default_lock_version_to_zero_for_issues.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class DefaultLockVersionToZeroForIssues < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    with_lock_retries do
+      change_column_default :issues, :lock_version, from: nil, to: 0
+    end
+  end
+end

db/migrate/20200203025619_default_lock_version_to_zero_for_epics.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class DefaultLockVersionToZeroForEpics < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    with_lock_retries do
+      change_column_default :epics, :lock_version, from: nil, to: 0
+    end
+  end
+end

db/migrate/20200203025744_default_lock_version_to_zero_for_ci_builds.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class DefaultLockVersionToZeroForCiBuilds < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    with_lock_retries do
+      change_column_default :ci_builds, :lock_version, from: nil, to: 0
+    end
+  end
+end

db/migrate/20200203025801_default_lock_version_to_zero_for_ci_stages.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class DefaultLockVersionToZeroForCiStages < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    with_lock_retries do
+      change_column_default :ci_stages, :lock_version, from: nil, to: 0
+    end
+  end
+end

db/migrate/20200203025821_default_lock_version_to_zero_for_ci_pipelines.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class DefaultLockVersionToZeroForCiPipelines < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    with_lock_retries do
+      change_column_default :ci_pipelines, :lock_version, from: nil, to: 0
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2020_01_30_161817) do
+ActiveRecord::Schema.define(version: 2020_02_03_025821) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -659,7 +659,7 @@ ActiveRecord::Schema.define(version: 2020_01_30_161817) do
     t.text "yaml_variables"
     t.datetime "queued_at"
     t.string "token"
-    t.integer "lock_version"
+    t.integer "lock_version", default: 0
     t.string "coverage_regex"
     t.integer "auto_canceled_by_id"
     t.boolean "retried"
@@ -835,7 +835,7 @@ ActiveRecord::Schema.define(version: 2020_01_30_161817) do
     t.datetime "finished_at"
     t.integer "duration"
     t.integer "user_id"
-    t.integer "lock_version"
+    t.integer "lock_version", default: 0
     t.integer "auto_canceled_by_id"
     t.integer "pipeline_schedule_id"
     t.integer "source"
@@ -949,7 +949,7 @@ ActiveRecord::Schema.define(version: 2020_01_30_161817) do
     t.datetime "updated_at"
     t.string "name"
     t.integer "status"
-    t.integer "lock_version"
+    t.integer "lock_version", default: 0
     t.integer "position"
     t.index ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", unique: true
     t.index ["pipeline_id", "position"], name: "index_ci_stages_on_pipeline_id_and_position"
@@ -1523,7 +1523,7 @@ ActiveRecord::Schema.define(version: 2020_01_30_161817) do
     t.integer "cached_markdown_version"
     t.integer "updated_by_id"
     t.integer "last_edited_by_id"
-    t.integer "lock_version"
+    t.integer "lock_version", default: 0
     t.date "start_date"
     t.date "end_date"
     t.datetime "last_edited_at"
@@ -2140,7 +2140,7 @@ ActiveRecord::Schema.define(version: 2020_01_30_161817) do
     t.boolean "confidential", default: false, null: false
     t.date "due_date"
     t.integer "moved_to_id"
-    t.integer "lock_version"
+    t.integer "lock_version", default: 0
     t.text "title_html"
     t.text "description_html"
     t.integer "time_estimate"
@@ -2562,7 +2562,7 @@ ActiveRecord::Schema.define(version: 2020_01_30_161817) do
     t.integer "approvals_before_merge"
     t.string "rebase_commit_sha"
     t.string "in_progress_merge_commit_sha"
-    t.integer "lock_version"
+    t.integer "lock_version", default: 0
     t.text "title_html"
     t.text "description_html"
     t.integer "time_estimate"

doc/development/sidekiq_style_guide.md

@@ -276,6 +276,125 @@ class SomeCrossCuttingConcernWorker
 end
 ```
+## Worker context
+
+To have some more information about workers in the logs, we add
+[metadata to the jobs in the form of an
+`ApplicationContext`](logging.md#logging-context-metadata-through-rails-or-grape-requests).
+In most cases, when scheduling a job from a request, this context will
+already be deducted from the request and added to the scheduled
+job.
+
+When a job runs, the context that was active when it was scheduled
+will be restored. This causes the context to be propagated to any job
+scheduled from within the running job.
+
+All this means that in most cases, to add context to jobs, we don't
+need to do anything.
+
+There are however some instances when there would be no context
+present when the job is scheduled, or the context that is present is
+likely to be incorrect. For these instances we've added rubocop-rules
+to draw attention and avoid incorrect metadata in our logs.
+
+As with most our cops, there are perfectly valid reasons for disabling
+them. In this case it could be that the context from the request is
+correct. Or maybe you've specified a context already in a way that
+isn't picked up by the cops. In any case, please leave a code-comment
+pointing to which context will be used when disabling the cops.
+
+When you do provide objects to the context, please make sure that the
+route for namespaces and projects is preloaded. This can be done using
+the `.with_route` scope defined on all `Routable`s.
+
+### Cron-Workers
+
+The context is automatically cleared for workers in the cronjob-queue
+(which `include CronjobQueue`), even when scheduling them from
+requests. We do this to avoid incorrect metadata when other jobs are
+scheduled from the cron-worker.
+
+Cron-Workers themselves run instance wide, so they aren't scoped to
+users, namespaces, projects or other resources that should be added to
+the context.
+
+However, they often schedule other jobs that _do_ require context.
+
+That is why there needs to be an indication of context somewhere in
+the worker. This can be done by using one of the following methods
+somewhere within the worker:
+
+1. Wrap the code that schedules jobs in the `with_context` helper:
+
+```ruby
+  def perform
+    deletion_cutoff = Gitlab::CurrentSettings
+                        .deletion_adjourned_period.days.ago.to_date
+    projects = Project.with_route.with_namespace
+                 .aimed_for_deletion(deletion_cutoff)
+
+    projects.find_each(batch_size: 100).with_index do |project, index|
+      delay = index * INTERVAL
+
+      with_context(project: project) do
+        AdjournedProjectDeletionWorker.perform_in(delay, project.id)
+      end
+    end
+  end
+```
+
+1. Use the a batch scheduling method that provides context:
+
+```ruby
+  def schedule_projects_in_batch(projects)
+    ProjectImportScheduleWorker.bulk_perform_async_with_contexts(
+      projects,
+      arguments_proc: -> (project) { project.id },
+      context_proc: -> (project) { { project: project } }
+    )
+  end
+```
+
+or when scheduling with delays:
+
+```ruby
+  diffs.each_batch(of: BATCH_SIZE) do |diffs, index|
+    DeleteDiffFilesWorker
+      .bulk_perform_in_with_contexts(index *  5.minutes,
+                                     diffs,
+                                     arguments_proc: -> (diff) { diff.id },
+                                     context_proc: -> (diff) { { project: diff.merge_request.target_project } })
+  end
+```
+
+### Jobs scheduled in bulk
+
+Often, when scheduling jobs in bulk, these jobs should have a separate
+context rather than the overarching context.
+
+If that is the case, `bulk_perform_async` can be replaced by the
+`bulk_perform_async_with_context` helper, and instead of
+`bulk_perform_in` use `bulk_perform_in_with_context`.
+
+For example:
+
+```ruby
+    ProjectImportScheduleWorker.bulk_perform_async_with_contexts(
+      projects,
+      arguments_proc: -> (project) { project.id },
+      context_proc: -> (project) { { project: project } }
+    )
+```
+
+Each object from the enumerable in the first argument is yielded into 2
+blocks:
+
+The `arguments_proc` which needs to return the list of arguments the
+job needs to be scheduled with.
+
+The `context_proc` which needs to return a hash with the context
+information for the job.
+
 ## Tests
 Each Sidekiq worker must be tested using RSpec, just like any other class. These

lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml

-# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_management/
-#
-# Configure the scanning tool through the environment variables.
-# List of the variables: https://gitlab.com/gitlab-org/security-products/license-management#settings
-# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+# Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/14624
+# Please, use License-Scanning.gitlab-ci.yml template instead
 variables:
   LICENSE_MANAGEMENT_SETUP_CMD: ''  # If needed, specify a command to setup your environment with a custom package manager.
@@ -16,6 +13,7 @@ license_management:
     SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD
   allow_failure: true
   script:
+    - echo "This template is deprecated, please use License-Scanning.gitlab-ci.yml template instead."
     - /run.sh analyze .
   artifacts:
     reports:

lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml

+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_compliance/
+#
+# Configure the scanning tool through the environment variables.
+# List of the variables: https://gitlab.com/gitlab-org/security-products/license-management#settings
+# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+
+variables:
+  LICENSE_MANAGEMENT_SETUP_CMD: ''  # If needed, specify a command to setup your environment with a custom package manager.
+
+license_scanning:
+  stage: test
+  image:
+    name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable"
+    entrypoint: [""]
+  variables:
+    SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD
+  allow_failure: true
+  script:
+    - /run.sh analyze .
+  after_script:
+    - mv gl-license-management-report.json gl-license-scanning-report.json
+  artifacts:
+    reports:
+      license_scanning: gl-license-scanning-report.json
+  dependencies: []
+  only:
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\blicense_management\b/
+  except:
+    variables:
+      - $LICENSE_MANAGEMENT_DISABLED

lib/gitlab/diff/formatters/image_formatter.rb

@@ -37,7 +37,9 @@ module Gitlab
         def ==(other)
           other.is_a?(self.class) &&
             x == other.x &&
-            y == other.y
+            y == other.y &&
+            width == other.width &&
+            height == other.height
         end
       end
     end

lib/gitlab/file_hook.rb

@@ -17,7 +17,7 @@ module Gitlab
     def self.execute_all_async(data)
       args = files.map { |file| [file, data] }
-      FileHookWorker.bulk_perform_async(args)
+      FileHookWorker.bulk_perform_async(args) # rubocop:disable Scalability/BulkPerformWithContext
     end
     def self.execute(file, data)