Add latest changes from gitlab-org/gitlab@master

doc/api/graphql/reference/index.md

@@ -923,6 +923,7 @@ Autogenerated return type of RemoveAwardEmoji
 | ---   |  ---- | ----------  |
 | `count` | Int! | Count of occurrences |
 | `culprit` | String! | Culprit of the error |
+| `externalBaseUrl` | String! | External Base URL of the Sentry Instance |
 | `externalUrl` | String! | External URL of the error |
 | `firstReleaseLastCommit` | String | Commit the error was first seen |
 | `firstReleaseShortVersion` | String | Release version the error was first seen |
@@ -930,6 +931,7 @@ Autogenerated return type of RemoveAwardEmoji
 | `frequency` | SentryErrorFrequency! => Array | Last 24hr stats of the error |
 | `gitlabCommit` | String | GitLab commit SHA attributed to the Error based on the release version |
 | `gitlabCommitPath` | String | Path to the GitLab page for the GitLab commit attributed to the error |
+| `gitlabIssuePath` | String | URL of GitLab Issue |
 | `id` | ID! | ID (global ID) of the error |
 | `lastReleaseLastCommit` | String | Commit the error was last seen |
 | `lastReleaseShortVersion` | String | Release version the error was last seen |
@@ -941,6 +943,7 @@ Autogenerated return type of RemoveAwardEmoji
 | `sentryProjectSlug` | String! | Slug of the project affected by the error |
 | `shortId` | String! | Short ID (Sentry ID) of the error |
 | `status` | SentryErrorStatus! | Status of the error |
+| `tags` | SentryErrorTags! | Tags associated with the Sentry Error |
 | `title` | String! | Title of the error |
 | `type` | String! | Type of the error |
 | `userCount` | Int! | Count of users affected by the error |
@@ -952,6 +955,15 @@ Autogenerated return type of RemoveAwardEmoji
 | `count` | Int! | Count of errors received since the previously recorded time |
 | `time` | Time! | Time the error frequency stats were recorded |
+## SentryErrorTags
+
+State of a Sentry error
+
+| Name  | Type  | Description |
+| ---   |  ---- | ----------  |
+| `level` | String | Severity level of the Sentry Error |
+| `logger` | String | Logger of the Sentry Error |
+
 ## Snippet
 Represents a snippet entry

doc/ci/variables/README.md

@@ -292,6 +292,8 @@ export CI_RUNNER_TAGS="docker, linux"
 export CI_SERVER="yes"
 export CI_SERVER_URL="https://example.com"
 export CI_SERVER_HOST="example.com"
+export CI_SERVER_PORT="443"
+export CI_SERVER_PROTOCOL="https"
 export CI_SERVER_NAME="GitLab"
 export CI_SERVER_REVISION="70606bf"
 export CI_SERVER_VERSION="8.9.0"
@@ -686,6 +688,10 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then
 ++ CI_SERVER_URL=https://gitlab.com:3000
 ++ export CI_SERVER_HOST=gitlab.com
 ++ CI_SERVER_HOST=gitlab.com
+++ export CI_SERVER_PORT=3000
+++ CI_SERVER_PORT=3000
+++ export CI_SERVER_PROTOCOL=https
+++ CI_SERVER_PROTOCOL=https
 ++ export CI_SERVER_NAME=GitLab
 ++ CI_SERVER_NAME=GitLab
 ++ export CI_SERVER_VERSION=12.6.0-pre

doc/ci/variables/predefined_variables.md

@@ -114,6 +114,8 @@ future GitLab releases.**
 | `CI_SERVER`                                   | all    | all    | Mark that job is executed in CI environment                                                                                                                                                                                                                                                                                                                |
 | `CI_SERVER_URL`                               | 12.7   | all    | The base URL of the GitLab instance, including protocol and port (like `https://gitlab.example.com:8080`)                                                                                                                                                                                                                                                           |
 | `CI_SERVER_HOST`                              | 12.1   | all    | Host component of the GitLab instance URL, without protocol and port (like `gitlab.example.com`)                                                                                                                                                                                                                                                           |
+| `CI_SERVER_PORT`                              | 12.8   | all    | Port component of the GitLab instance URL, without host and protocol (like `3000`)                                                                                                                                                                                                                                                                         |
+| `CI_SERVER_PROTOCOL`                          | 12.8   | all    | Protocol component of the GitLab instance URL, without host and port (like `https`)                                                                                                                                                                                                                                                                        |
 | `CI_SERVER_NAME`                              | all    | all    | The name of CI server that is used to coordinate jobs                                                                                                                                                                                                                                                                                                      |
 | `CI_SERVER_REVISION`                          | all    | all    | GitLab revision that is used to schedule jobs                                                                                                                                                                                                                                                                                                              |
 | `CI_SERVER_VERSION`                           | all    | all    | GitLab version that is used to schedule jobs                                                                                                                                                                                                                                                                                                               |

doc/user/discussions/index.md

@@ -439,6 +439,16 @@ Suggestions covering multiple lines are limited to 100 lines _above_ and 100
 lines _below_ the commented diff line, allowing up to 200 changed lines per
 suggestion.
+### Code block nested in Suggestions
+
+If you need to make a suggestion that involves a
+[fenced code block](../markdown.md#code-spans-and-blocks), wrap your suggestion in four backticks
+instead of the usual three.
+
+![A comment editor with a suggestion with a fenced code block](img/suggestion_code_block_editor_v12_8.png)
+
+![Ouput of a comment with a suggestion with a fenced code block](img/suggestion_code_block_output_v12_8.png)
+
 ### Configure the commit message for applied Suggestions
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13086) in GitLab 12.7.

lib/api/broadcast_messages.rb

@@ -4,9 +4,6 @@ module API
   class BroadcastMessages < Grape::API
     include PaginationParams
-    before { authenticate! }
-    before { authenticated_as_admin! }
-
     resource :broadcast_messages do
       helpers do
         def find_message
@@ -40,6 +37,8 @@ module API
         optional :target_path, type: String, desc: 'Target path'
       end
       post do
+        authenticated_as_admin!
+
         message = BroadcastMessage.create(declared_params(include_missing: false))
         if message.persisted?
@@ -76,6 +75,8 @@ module API
         optional :target_path, type: String, desc: 'Target path'
       end
       put ':id' do
+        authenticated_as_admin!
+
         message = find_message
         if message.update(declared_params(include_missing: false))
@@ -93,6 +94,8 @@ module API
         requires :id, type: Integer, desc: 'Broadcast message ID'
       end
       delete ':id' do
+        authenticated_as_admin!
+
         message = find_message
         destroy_conditionally!(message)

lib/api/helpers/internal_helpers.rb

@@ -104,14 +104,12 @@ module API
       # rubocop:disable Gitlab/ModuleWithInstanceVariables
       def set_project
-        if params[:gl_repository]
-          @project, @repo_type = Gitlab::GlRepository.parse(params[:gl_repository])
-          @redirected_path = nil
-        elsif params[:project]
-          @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse(params[:project])
-        else
-          @project, @repo_type, @redirected_path = nil, nil, nil
-        end
+        @project, @repo_type, @redirected_path =
+          if params[:gl_repository]
+            Gitlab::GlRepository.parse(params[:gl_repository])
+          elsif params[:project]
+            Gitlab::RepoPath.parse(params[:project])
+          end
       end
       # rubocop:enable Gitlab/ModuleWithInstanceVariables

lib/constraints/project_url_constrainer.rb

@@ -4,7 +4,7 @@ module Constraints
   class ProjectUrlConstrainer
     def matches?(request, existence_check: true)
       namespace_path = request.params[:namespace_id]
-      project_path = request.params[:project_id] || request.params[:id]
+      project_path = request.params[:project_id] || request.params[:id] || request.params[:repository_id]
       full_path = [namespace_path, project_path].join('/')
       return false unless ProjectPathValidator.valid_path?(full_path)

lib/feature/gitaly.rb

@@ -9,7 +9,6 @@ class Feature
       %w[
         cache_invalidator
         inforef_uploadpack_cache
-        get_tag_messages_go
         filter_shas_with_signatures_go
         commit_without_batch_check
       ].freeze

lib/gitaly/server.rb

@@ -53,6 +53,20 @@ module Gitaly
       storage_status&.fs_type
     end
+    def disk_used
+      disk_statistics_storage_status&.used
+    end
+
+    def disk_available
+      disk_statistics_storage_status&.available
+    end
+
+    # Simple convenience method for when obtaining both used and available
+    # statistics at once is preferred.
+    def disk_stats
+      disk_statistics_storage_status
+    end
+
     def address
       Gitlab::GitalyClient.address(@storage)
     rescue RuntimeError => e
@@ -65,6 +79,10 @@ module Gitaly
       @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage }
     end
+    def disk_statistics_storage_status
+      @disk_statistics_storage_status ||= disk_statistics.storage_statuses.find { |s| s.storage_name == storage }
+    end
+
     def matches_sha?
       match = server_version.match(SHA_VERSION_REGEX)
       return false unless match
@@ -76,7 +94,19 @@ module Gitaly
       @info ||=
         begin
           Gitlab::GitalyClient::ServerService.new(@storage).info
-        rescue GRPC::Unavailable, GRPC::DeadlineExceeded
+        rescue GRPC::Unavailable, GRPC::DeadlineExceeded => ex
+          Gitlab::ErrorTracking.track_exception(ex)
+          # This will show the server as being out of date
+          Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: [])
+        end
+    end
+
+    def disk_statistics
+      @disk_statistics ||=
+        begin
+          Gitlab::GitalyClient::ServerService.new(@storage).disk_statistics
+        rescue GRPC::Unavailable, GRPC::DeadlineExceeded => ex
+          Gitlab::ErrorTracking.track_exception(ex)
           # This will show the server as being out of date
           Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: [])
         end

lib/gitlab/gitaly_client.rb

@@ -432,10 +432,7 @@ module Gitlab
     end
     def self.filesystem_id(storage)
-      response = Gitlab::GitalyClient::ServerService.new(storage).info
-      storage_status = response.storage_statuses.find { |status| status.storage_name == storage }
-
-      storage_status&.filesystem_id
+      Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id
     end
     def self.filesystem_id_from_disk(storage)
@@ -446,6 +443,14 @@ module Gitlab
       nil
     end
+    def self.filesystem_disk_available(storage)
+      Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available
+    end
+
+    def self.filesystem_disk_used(storage)
+      Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used
+    end
+
     def self.timeout(timeout_name)
       Gitlab::CurrentSettings.current_application_settings[timeout_name]
     end

lib/gitlab/gitaly_client/server_service.rb

@@ -13,6 +13,24 @@ module Gitlab
       def info
         GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new, timeout: GitalyClient.fast_timeout)
       end
+
+      def disk_statistics
+        GitalyClient.call(@storage, :server_service, :disk_statistics, Gitaly::DiskStatisticsRequest.new, timeout: GitalyClient.fast_timeout)
+      end
+
+      def storage_info
+        storage_specific(info)
+      end
+
+      def storage_disk_statistics
+        storage_specific(disk_statistics)
+      end
+
+      private
+
+      def storage_specific(response)
+        response.storage_statuses.find { |status| status.storage_name == @storage }
+      end
     end
   end
 end

lib/gitlab/gon_helper.rb

@@ -43,6 +43,9 @@ module Gitlab
       # Initialize gon.features with any flags that should be
       # made globally available to the frontend
       push_frontend_feature_flag(:snippets_vue, default_enabled: false)
+      push_frontend_feature_flag(:monaco_snippets, default_enabled: false)
+      push_frontend_feature_flag(:monaco_blobs, default_enabled: false)
+      push_frontend_feature_flag(:monaco_ci, default_enabled: false)
     end
     # Exposes the state of a feature flag to the frontend code.

lib/gitlab/import_export/import_failure_service.rb

@@ -12,9 +12,14 @@ module Gitlab
         @association = importable.association(:import_failures)
       end
-      def with_retry(relation_key, relation_index)
+      def with_retry(action:, relation_key: nil, relation_index: nil)
         on_retry = -> (exception, retry_count, *_args) do
-          log_import_failure(relation_key, relation_index, exception, retry_count)
+          log_import_failure(
+            source: action,
+            relation_key: relation_key,
+            relation_index: relation_index,
+            exception: exception,
+            retry_count: retry_count)
         end
         Retriable.with_context(:relation_import, on_retry: on_retry) do
@@ -22,8 +27,9 @@ module Gitlab
         end
       end
-      def log_import_failure(relation_key, relation_index, exception, retry_count = 0)
+      def log_import_failure(source:, relation_key: nil, relation_index: nil, exception:, retry_count: 0)
         extra = {
+          source: source,
           relation_key: relation_key,
           relation_index: relation_index,
           retry_count: retry_count

lib/gitlab/import_export/project_tree_restorer.rb

@@ -21,7 +21,9 @@ module Gitlab
         RelationRenameService.rename(@tree_hash)
         if relation_tree_restorer.restore
-          @project.merge_requests.set_latest_merge_request_diff_ids!
+          import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
+            @project.merge_requests.set_latest_merge_request_diff_ids!
+          end
           true
         else
@@ -72,6 +74,10 @@ module Gitlab
       def reader
         @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
       end
+
+      def import_failure_service
+        @import_failure_service ||= ImportFailureService.new(@project)
+      end
     end
   end
 end

lib/gitlab/import_export/relation_tree_restorer.rb

@@ -73,13 +73,17 @@ module Gitlab
         relation_object.assign_attributes(importable_class_sym => @importable)
-        import_failure_service.with_retry(relation_key, relation_index) do
+        import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do
           relation_object.save!
         end
         save_id_mapping(relation_key, data_hash, relation_object)
       rescue => e
-        import_failure_service.log_import_failure(relation_key, relation_index, e)
+        import_failure_service.log_import_failure(
+          source: 'process_relation_item!',
+          relation_key: relation_key,
+          relation_index: relation_index,
+          exception: e)
       end
       def import_failure_service

lib/gitlab/middleware/read_only/controller.rb

@@ -12,12 +12,12 @@ module Gitlab
         ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'
         WHITELISTED_GIT_ROUTES = {
-          'projects/git_http' => %w{git_upload_pack git_receive_pack}
+          'repositories/git_http' => %w{git_upload_pack git_receive_pack}
         }.freeze
         WHITELISTED_GIT_LFS_ROUTES = {
-          'projects/lfs_api' => %w{batch},
-          'projects/lfs_locks_api' => %w{verify create unlock}
+          'repositories/lfs_api' => %w{batch},
+          'repositories/lfs_locks_api' => %w{verify create unlock}
         }.freeze
         WHITELISTED_GIT_REVISION_ROUTES = {

lib/gitlab/sidekiq_config.rb

@@ -6,6 +6,7 @@ module Gitlab
   module SidekiqConfig
     FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml'
     EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml'
+    SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml'
     QUEUE_CONFIG_PATHS = [
       FOSS_QUEUE_CONFIG_PATH,
@@ -13,11 +14,19 @@ module Gitlab
     ].compact.freeze
     # For queues that don't have explicit workers - default and mailers
-    DummyWorker = Struct.new(:queue)
+    DummyWorker = Struct.new(:queue, :weight) do
+      def queue_namespace
+        nil
+      end
+
+      def get_weight
+        weight
+      end
+    end
     DEFAULT_WORKERS = [
-      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default'), ee: false),
-      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers'), ee: false)
+      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default', 1), ee: false),
+      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers', 2), ee: false)
     ].freeze
     class << self
@@ -30,7 +39,7 @@ module Gitlab
       def config_queues
         @config_queues ||= begin
-          config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml'))
+          config = YAML.load_file(Rails.root.join(SIDEKIQ_QUEUES_PATH))
           config[:queues].map(&:first)
         end
       end
@@ -65,6 +74,28 @@ module Gitlab
         Gitlab.ee? && ee_workers != YAML.safe_load(File.read(EE_QUEUE_CONFIG_PATH))
       end
+      def queues_for_sidekiq_queues_yml
+        namespaces_with_equal_weights =
+          workers
+            .group_by(&:queue_namespace)
+            .map(&:last)
+            .select { |workers| workers.map(&:get_weight).uniq.count == 1 }
+            .map(&:first)
+
+        namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set
+        remaining_queues = workers.reject { |worker| namespaces.include?(worker.queue_namespace) }
+
+        (namespaces_with_equal_weights.map(&:namespace_and_weight) +
+         remaining_queues.map(&:queue_and_weight)).sort
+      end
+
+      def sidekiq_queues_yml_outdated?
+        # YAML.load is OK here as we control the file contents
+        config_queues = YAML.load(File.read(SIDEKIQ_QUEUES_PATH))[:queues] # rubocop:disable Security/YAMLLoad
+
+        queues_for_sidekiq_queues_yml != config_queues
+      end
+
       private
       def find_workers(root, ee:)

lib/gitlab/sidekiq_config/worker.rb

@@ -7,8 +7,9 @@ module Gitlab
       attr_reader :klass
       delegate :feature_category_not_owned?, :get_feature_category,
-               :get_worker_resource_boundary, :latency_sensitive_worker?,
-               :queue, :worker_has_external_dependencies?,
+               :get_weight, :get_worker_resource_boundary,
+               :latency_sensitive_worker?, :queue, :queue_namespace,
+               :worker_has_external_dependencies?,
                to: :klass
       def initialize(klass, ee:)
@@ -35,7 +36,7 @@ module Gitlab
       # Put namespaced queues first
       def to_sort
-        [queue.include?(':') ? 0 : 1, queue]
+        [queue_namespace ? 0 : 1, queue]
       end
       # YAML representation
@@ -46,6 +47,14 @@ module Gitlab
       def to_yaml
         queue
       end
+
+      def namespace_and_weight
+        [queue_namespace, get_weight]
+      end
+
+      def queue_and_weight
+        [queue, get_weight]
+      end
     end
   end
 end