Add latest changes from gitlab-org/gitlab@master

app/uploaders/file_uploader.rb

@@ -16,6 +16,9 @@ class FileUploader < GitlabUploader
   MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)}.freeze
   DYNAMIC_PATH_PATTERN = %r{.*(?<secret>\h{32})/(?<identifier>.*)}.freeze
+  VALID_SECRET_PATTERN = %r{\A\h{10,32}\z}.freeze
+
+  InvalidSecret = Class.new(StandardError)
   after :remove, :prune_store_dir
@@ -153,6 +156,10 @@ class FileUploader < GitlabUploader
   def secret
     @secret ||= self.class.generate_secret
+
+    raise InvalidSecret unless @secret =~ VALID_SECRET_PATTERN
+
+    @secret
   end
   # return a new uploader with a file copy on another project

app/validators/addressable_url_validator.rb

@@ -23,7 +23,8 @@
 # protect against Server-side Request Forgery (SSRF), or check for the right port.
 #
 # Configuration options:
-# * <tt>message</tt> - A custom error message (default is: "must be a valid URL").
+# * <tt>message</tt> - A custom error message, used when the URL is blank. (default is: "must be a valid URL").
+# * <tt>blocked_message</tt> - A custom error message, used when the URL is blocked. Default: +'is blocked: %{exception_message}'+.
 # * <tt>schemes</tt> - Array of URI schemes. Default: +['http', 'https']+
 # * <tt>allow_localhost</tt> - Allow urls pointing to +localhost+. Default: +true+
 # * <tt>allow_local_network</tt> - Allow urls pointing to private network addresses. Default: +true+
@@ -59,7 +60,8 @@ class AddressableUrlValidator < ActiveModel::EachValidator
   }.freeze
   DEFAULT_OPTIONS = BLOCKER_VALIDATE_OPTIONS.merge({
-    message: 'must be a valid URL'
+    message: 'must be a valid URL',
+    blocked_message: 'is blocked: %{exception_message}'
   }).freeze
   def initialize(options)
@@ -80,7 +82,7 @@ class AddressableUrlValidator < ActiveModel::EachValidator
     Gitlab::UrlBlocker.validate!(value, blocker_args)
   rescue Gitlab::UrlBlocker::BlockedUrlError => e
-    record.errors.add(attribute, "is blocked: #{e.message}")
+    record.errors.add(attribute, options.fetch(:blocked_message) % { exception_message: e.message })
   end
   private

app/views/admin/application_settings/_grafana.html.haml

-= form_for @application_setting, url: admin_application_settings_path(anchor: 'js-grafana-settings'), html: { class: 'fieldset-form' } do |f|
+= form_for @application_setting, url: metrics_and_profiling_admin_application_settings_path(anchor: 'js-grafana-settings'), html: { class: 'fieldset-form' } do |f|
   = form_errors(@application_setting)
   %fieldset

app/views/clusters/clusters/_form.html.haml

@@ -3,7 +3,7 @@
   .form-group
     %h5= s_('ClusterIntegration|Integration status')
     %label.append-bottom-0.js-cluster-enable-toggle-area
-      = render "shared/buttons/project_feature_toggle", is_checked: @cluster.enabled?, label: s_("ClusterIntegration|Toggle Kubernetes cluster"), disabled: !can?(current_user, :update_cluster, @cluster) do
+      = render "shared/buttons/project_feature_toggle", is_checked: @cluster.enabled?, label: s_("ClusterIntegration|Toggle Kubernetes cluster"), disabled: !can?(current_user, :update_cluster, @cluster), data: { qa_selector: 'integration_status_toggle' } do
         = field.hidden_field :enabled, { class: 'js-project-feature-toggle-input'}
       .form-text.text-muted= s_('ClusterIntegration|Enable or disable GitLab\'s connection to your Kubernetes cluster.')
@@ -22,7 +22,7 @@
   .form-group
     %h5= s_('ClusterIntegration|Base domain')
-    = field.text_field :base_domain, class: 'col-md-6 form-control js-select-on-focus qa-base-domain'
+    = field.text_field :base_domain, class: 'col-md-6 form-control js-select-on-focus', data: { qa_selector: 'base_domain_field' }
     .form-text.text-muted
       - auto_devops_url = help_page_path('topics/autodevops/index')
       - auto_devops_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: auto_devops_url }
@@ -37,4 +37,4 @@
   - if can?(current_user, :update_cluster, @cluster)
     .form-group
-      = field.submit _('Save changes'), class: 'btn btn-success qa-save-domain'
+      = field.submit _('Save changes'), class: 'btn btn-success', data: { qa_selector: 'save_changes_button' }

app/views/clusters/clusters/user/_form.html.haml

@@ -54,4 +54,4 @@
       = render('clusters/clusters/namespace', platform_field: platform_kubernetes_field)
   .form-group
-    = field.submit s_('ClusterIntegration|Add Kubernetes cluster'), class: 'btn btn-success'
+    = field.submit s_('ClusterIntegration|Add Kubernetes cluster'), class: 'btn btn-success', data: { qa_selector: 'add_kubernetes_cluster_button' }

app/views/layouts/nav/sidebar/_admin.html.haml

@@ -83,7 +83,7 @@
                 = _('Requests Profiles')
           - if Gitlab::CurrentSettings.current_application_settings.grafana_enabled?
             = nav_link do
-              = link_to Gitlab::CurrentSettings.current_application_settings.grafana_url, target: '_blank', title: _('Metrics Dashboard') do
+              = link_to Gitlab::CurrentSettings.current_application_settings.grafana_url, target: '_blank', title: _('Metrics Dashboard'), rel: 'noopener noreferrer' do
                 %span
                   = _('Metrics Dashboard')
           = render_if_exists 'layouts/nav/ee/admin/new_monitoring_sidebar'

app/views/shared/issuable/form/_branch_chooser.html.haml

@@ -8,7 +8,9 @@
 .form-group.row.d-flex.gl-pl-3.gl-pr-3.branch-selector
   .align-self-center
-    %span= s_('From %{source_title} into').html_safe % { source_title: "<code>#{source_title}</code>".html_safe }
+    %span
+      = _('From <code>%{source_title}</code> into').html_safe % { source_title: source_title }
+
     - if issuable.new_record?
       %code= target_title
       &nbsp;

changelogs/unreleased/208675-add-package_name-as-option-to-packages-api.yml

+---
+title: Added package_name as filter parameter to packages API
+merge_request: 26291
+author:
+type: added

changelogs/unreleased/georgekoltsov-27883-fix-import-pipeline-order.yml

+---
+title: Fix reversed pipeline order on Project Import
+merge_request: 26390
+author:
+type: fixed

config/initializers/8_devise.rb

@@ -80,8 +80,16 @@ Devise.setup do |config|
   # When allow_unconfirmed_access_for is zero, the user won't be able to sign in without confirming.
   # You can use this to let your user access some features of your application
   # without confirming the account, but blocking it after a certain period
-  # (ie 2 days).
-  config.allow_unconfirmed_access_for = 30.days
+  # (e.g. 3 days).
+  config.allow_unconfirmed_access_for = 3.days
+
+  # A period that the user is allowed to confirm their account before their
+  # token becomes invalid. For example, if set to 1.day, the user can confirm
+  # their account within 1 days after the mail was sent, but on the second day
+  # their account can't be confirmed with the token any more.
+  # Default is nil, meaning there is no restriction on how long a user can take
+  # before confirming their account.
+  config.confirm_within = 1.day
   # Defines which key will be used when confirming an account
   # config.confirmation_keys = [ :email ]

config/initializers/sidekiq.rb

@@ -36,15 +36,6 @@ use_sidekiq_legacy_memory_killer = !use_sidekiq_daemon_memory_killer
 use_request_store = ENV.fetch('SIDEKIQ_REQUEST_STORE', 1).to_i.nonzero?
 Sidekiq.configure_server do |config|
-  config.redis = queues_config_hash
-
-  config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator({
-    metrics: Settings.monitoring.sidekiq_exporter,
-    arguments_logger: ENV['SIDEKIQ_LOG_ARGUMENTS'] && !enable_json_logs,
-    memory_killer: enable_sidekiq_memory_killer && use_sidekiq_legacy_memory_killer,
-    request_store: use_request_store
-  }))
-
   if enable_json_logs
     Sidekiq.logger.formatter = Gitlab::SidekiqLogging::JSONFormatter.new
     config.options[:job_logger] = Gitlab::SidekiqLogging::StructuredLogger
@@ -54,6 +45,15 @@ Sidekiq.configure_server do |config|
     config.error_handlers << Gitlab::SidekiqLogging::ExceptionHandler.new
   end
+  config.redis = queues_config_hash
+
+  config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator({
+    metrics: Settings.monitoring.sidekiq_exporter,
+    arguments_logger: ENV['SIDEKIQ_LOG_ARGUMENTS'] && !enable_json_logs,
+    memory_killer: enable_sidekiq_memory_killer && use_sidekiq_legacy_memory_killer,
+    request_store: use_request_store
+  }))
+
   config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)
   config.on :startup do

db/migrate/20200214085940_clean_grafana_url.rb

+# frozen_string_literal: true
+
+class CleanGrafanaUrl < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    execute(
+      <<-SQL
+      UPDATE
+        application_settings
+      SET
+        grafana_url = default
+      WHERE
+        position('javascript:' IN btrim(application_settings.grafana_url)) = 1
+      SQL
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20200204113224_schedule_recalculate_project_authorizations_second_run.rb

+# frozen_string_literal: true
+
+class ScheduleRecalculateProjectAuthorizationsSecondRun < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'RecalculateProjectAuthorizationsWithMinMaxUserId'
+  BATCH_SIZE = 2_500
+  DELAY_INTERVAL = 2.minutes.to_i
+
+  disable_ddl_transaction!
+
+  class User < ActiveRecord::Base
+    include ::EachBatch
+
+    self.table_name = 'users'
+  end
+
+  def up
+    say "Scheduling #{MIGRATION} jobs"
+
+    User.each_batch(of: BATCH_SIZE) do |batch, index|
+      delay = index * DELAY_INTERVAL
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+      BackgroundMigrationWorker.perform_in(delay, MIGRATION, range)
+    end
+  end
+
+  def down
+  end
+end

doc/administration/high_availability/README.md

@@ -76,7 +76,7 @@ On different cloud vendors a best effort like for like can be used.
 - **Known Issues:** For the latest list of known performance issues head
 [here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
-| Service                     | Nodes | Configuration         | GCP type      |
+| Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
 | GitLab Rails[^1]            | 3     | 8 vCPU, 7.2GB Memory  | n1-highcpu-8 |
 | PostgreSQL                  | 3     | 2 vCPU, 7.5GB Memory  | n1-standard-2 |
@@ -98,7 +98,7 @@ On different cloud vendors a best effort like for like can be used.
 - **Known Issues:** For the latest list of known performance issues head
 [here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
-| Service                     | Nodes | Configuration         | GCP type      |
+| Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
 | GitLab Rails[^1]            | 3     | 16 vCPU, 14.4GB Memory | n1-highcpu-16 |
 | PostgreSQL                  | 3     | 2 vCPU, 7.5GB Memory  | n1-standard-2 |
@@ -120,7 +120,7 @@ On different cloud vendors a best effort like for like can be used.
 - **Known Issues:** For the latest list of known performance issues head
 [here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
-| Service                     | Nodes | Configuration         | GCP type      |
+| Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
 | GitLab Rails[^1]            | 3     | 32 vCPU, 28.8GB Memory | n1-highcpu-32 |
 | PostgreSQL                  | 3     | 4 vCPU, 15GB Memory   | n1-standard-4 |
@@ -145,9 +145,9 @@ On different cloud vendors a best effort like for like can be used.
 - **Known Issues:** For the latest list of known performance issues head
 [here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
-| Service                     | Nodes | Configuration         | GCP type      |
+| Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
-| GitLab Rails[^1]            | 7     | 32 vCPU, 28.8GB Memory | n1-highcpu-32 |
+| GitLab Rails[^1]            | 5     | 32 vCPU, 28.8GB Memory | n1-highcpu-32 |
 | PostgreSQL                  | 3     | 8 vCPU, 30GB Memory   | n1-standard-8 |
 | PgBouncer                   | 3     | 2 vCPU, 1.8GB Memory  | n1-highcpu-2  |
 | Gitaly[^2] [^5] [^7]        | X     | 32 vCPU, 120GB Memory | n1-standard-32 |
@@ -170,9 +170,9 @@ On different cloud vendors a best effort like for like can be used.
 - **Known Issues:** For the latest list of known performance issues head
 [here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
-| Service                     | Nodes | Configuration         | GCP type      |
+| Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
-| GitLab Rails[^1]            | 15    | 32 vCPU, 28.8GB Memory | n1-highcpu-32 |
+| GitLab Rails[^1]            | 12    | 32 vCPU, 28.8GB Memory | n1-highcpu-32 |
 | PostgreSQL                  | 3     | 16 vCPU, 60GB Memory  | n1-standard-16 |
 | PgBouncer                   | 3     | 2 vCPU, 1.8GB Memory  | n1-highcpu-2  |
 | Gitaly[^2] [^5] [^7]        | X     | 64 vCPU, 240GB Memory | n1-standard-64 |
@@ -230,3 +230,9 @@ On different cloud vendors a best effort like for like can be used.
       as with time they may be adjusted higher or lower depending on the scale of your
       environment's workload. If you're running the environment on a Cloud provider
       you may need to refer to their documentation on how configure IOPS correctly.
+
+[^8]: The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms)
+      CPU platform on GCP. On different hardware you may find that adjustments, either lower
+      or higher, are required for your CPU or Node counts accordingly. For more info a
+      [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found
+      [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks).

doc/administration/monitoring/performance/grafana_configuration.md

@@ -117,8 +117,9 @@ If you have set up Grafana, you can enable a link to access it easily from the s
 1. Expand **Metrics - Grafana**.
 1. Check the "Enable access to Grafana" checkbox.
 1. If Grafana is enabled through Omnibus GitLab and on the same server,
-   leave "Grafana URL" unchanged. In any other case, enter the full URL
-   path of the Grafana instance.
+   leave **Grafana URL** unchanged. It should be `/-/grafana`.
+
+   In any other case, enter the full URL of the Grafana instance.
 1. Click **Save changes**.
 1. The new link will be available in the **Admin Area > Monitoring > Metrics Dashboard**.

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -262,14 +262,6 @@ p.each do |project|
 end
 ```
-### Identify un-indexed projects
-
-```ruby
-Project.find_each do |project|
-  puts "id #{project.id}: #{project.namespace.name.to_s}/#{project.name.to_s}" if project.index_status.nil?
-end
-```
-
 ## Wikis
 ### Recreate
@@ -761,12 +753,6 @@ Ci::Pipeline.where(project_id: p.id).where(status: 'pending').each {|p| p.cancel
 Ci::Pipeline.where(project_id: p.id).where(status: 'pending').count
 ```
-### Manually modify runner minutes
-
-```ruby
-Namespace.find_by_full_path("user/proj").namespace_statistics.update(shared_runners_seconds: 27360)
-```
-
 ### Remove artifacts more than a week old
 The Latest version of these steps can be found in the [job artifacts documentation](../job_artifacts.md)
@@ -806,21 +792,6 @@ build.dependencies.each do |d| { puts "status: #{d.status}, finished at: #{d.fin
   completed: #{d.complete?}, artifacts_expired: #{d.artifacts_expired?}, erased: #{d.erased?}" }
 ```
-### Disable strict artifact checking (Introduced in GitLab 10.3.0)
-
-See [job artifacts documentation](../job_artifacts.md#validation-for-dependencies).
-
-```ruby
-Feature.enable('ci_disable_validates_dependencies')
-```
-
-### Remove CI traces older than 6 months
-
-```ruby
-current_user = User.find_by_email('cindy@gitlap.com')
-Ci::Build.where("finished_at < ?", 6.months.ago.to_date).each {|b| puts b.id; b.erase(erased_by: current_user) if b.erasable?};nil
-```
-
 ### Try CI service
 ```ruby
@@ -965,12 +936,6 @@ end
 ## Sidekiq
-### Size of a queue
-
-```ruby
-Sidekiq::Queue.new('background_migration').size
-```
-
 ### Kill a worker's Sidekiq jobs
 ```ruby
@@ -1017,12 +982,6 @@ See <https://github.com/mperham/sidekiq/wiki/Signals#ttin>.
 /opt/gitlab/embedded/bin/redis-cli -s /var/opt/gitlab/redis/redis.socket
 ```
-### Connect to Redis (HA)
-
-```shell
-/opt/gitlab/embedded/bin/redis-cli -h <host ip> -a <password>
-```
-
 ## LFS
 ### Get info about LFS objects and associated project

doc/api/graphql/reference/gitlab_schema.graphql

@@ -1665,7 +1665,7 @@ type DiffRefs {
   """
   Merge base of the branch the comment was made on
   """
-  baseSha: String!
+  baseSha: String
   """
   SHA of the HEAD at the time the comment was made

doc/api/graphql/reference/gitlab_schema.json

@@ -8546,13 +8546,9 @@
               ],
               "type": {
-                "kind": "NON_NULL",
-                "name": null,
-                "ofType": {
-                  "kind": "SCALAR",
-                  "name": "String",
-                  "ofType": null
-                }
+                "kind": "SCALAR",
+                "name": "String",
+                "ofType": null
               },
               "isDeprecated": false,
               "deprecationReason": null

doc/api/graphql/reference/index.md

@@ -286,7 +286,7 @@ Autogenerated return type of DestroySnippet
 | Name  | Type  | Description |
 | ---   |  ---- | ----------  |
-| `baseSha` | String! | Merge base of the branch the comment was made on |
+| `baseSha` | String | Merge base of the branch the comment was made on |
 | `headSha` | String! | SHA of the HEAD at the time the comment was made |
 | `startSha` | String! | SHA of the branch being compared against |

doc/api/packages.md

@@ -21,6 +21,7 @@ GET /projects/:id/packages
 | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, or `type`. |
 | `sort`    | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. |
 | `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm` or `nuget`. (_Introduced in GitLab 12.9_)
+| `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_Introduced in GitLab 12.9_)
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/packages