Add latest changes from gitlab-org/gitlab@master

.gitlab/CODEOWNERS

@@ -32,4 +32,5 @@ lib/gitlab/github_import/ @gitlab-org/maintainers/database
 /.gitlab/ci/ @gl-quality/eng-prod
 Dangerfile @gl-quality/eng-prod
 /danger/ @gl-quality/eng-prod
+/lib/gitlab/danger/ @gl-quality/eng-prod
 /scripts/ @gl-quality/eng-prod

.gitlab/ci/review.gitlab-ci.yml

@@ -97,7 +97,10 @@ schedule:review-build-cng:
   variables:
     HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
     DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
-    GITLAB_HELM_CHART_REF: "v2.3.7"
+    # v2.3.7 + some stability improvements not yet released:
+    # - sidekiq readinessProbe should be `pgrep -f sidekiq`: https://gitlab.com/gitlab-org/charts/gitlab/merge_requests/991
+    # - Allows livenessProbe and readinessProbe to be configured for unicorn: https://gitlab.com/gitlab-org/charts/gitlab/merge_requests/985
+    GITLAB_HELM_CHART_REF: "df7c52dc69df441909880b8f2fd15e938cdb2047"
     GITLAB_EDITION: "ce"
   environment:
     name: review/${CI_COMMIT_REF_NAME}

.gitlab/issue_templates/Security developer workflow.md

@@ -29,7 +29,7 @@ Set the title to: `Description of the original issue`
 #### Documentation and final details
-- [ ] Check the topic on #security to see when the next release is going to happen and add a link to the [links section](#links)
+- [ ] Check the topic on #releases to see when the next release is going to happen and add a link to the [links section](#links)
 - [ ] Add links to this issue and your MRs in the description of the security release issue
 - [ ] Find out the versions affected (the Git history of the files affected may help you with this) and add them to the [details section](#details)
 - [ ] Fill in any upgrade notes that users may need to take into account in the [details section](#details)

app/services/create_branch_service.rb

@@ -14,7 +14,7 @@ class CreateBranchService < BaseService
     if new_branch
       success(new_branch)
     else
-      error('Invalid reference name')
+      error("Invalid reference name: #{branch_name}")
     end
   rescue Gitlab::Git::PreReceiveError => ex
     error(ex.message)

app/services/metrics/dashboard/grafana_metric_embed_service.rb

+# frozen_string_literal: true
+
+# Responsible for returning a gitlab-compatible dashboard
+# containing info based on a grafana dashboard and datasource.
+#
+# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
+module Metrics
+  module Dashboard
+    class GrafanaMetricEmbedService < ::Metrics::Dashboard::BaseService
+      include ReactiveCaching
+
+      SEQUENCE = [
+        ::Gitlab::Metrics::Dashboard::Stages::GrafanaFormatter
+      ].freeze
+
+      self.reactive_cache_key = ->(service) { service.cache_key }
+      self.reactive_cache_lease_timeout = 30.seconds
+      self.reactive_cache_refresh_interval = 30.minutes
+      self.reactive_cache_lifetime = 30.minutes
+      self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) }
+
+      class << self
+        # Determines whether the provided params are sufficient
+        # to uniquely identify a grafana dashboard.
+        def valid_params?(params)
+          [
+            params[:embedded],
+            params[:grafana_url]
+          ].all?
+        end
+
+        def from_cache(project_id, user_id, grafana_url)
+          project = Project.find(project_id)
+          user = User.find(user_id)
+
+          new(project, user, grafana_url: grafana_url)
+        end
+      end
+
+      def get_dashboard
+        with_reactive_cache(*cache_key) { |result| result }
+      end
+
+      # Inherits the primary logic from the parent class and
+      # maintains the service's API while including ReactiveCache
+      def calculate_reactive_cache(*)
+        ::Metrics::Dashboard::BaseService
+          .instance_method(:get_dashboard)
+          .bind(self)
+          .call() # rubocop:disable Style/MethodCallWithoutArgsParentheses
+      end
+
+      def cache_key(*args)
+        [project.id, current_user.id, grafana_url]
+      end
+
+      # Required for ReactiveCaching; Usage overridden by
+      # self.reactive_cache_worker_finder
+      def id
+        nil
+      end
+
+      private
+
+      def get_raw_dashboard
+        raise MissingIntegrationError unless client
+
+        grafana_dashboard = fetch_dashboard
+        datasource = fetch_datasource(grafana_dashboard)
+
+        params.merge!(grafana_dashboard: grafana_dashboard, datasource: datasource)
+
+        {}
+      end
+
+      def fetch_dashboard
+        uid = GrafanaUidParser.new(grafana_url, project).parse
+        raise DashboardProcessingError.new('Dashboard uid not found') unless uid
+
+        response = client.get_dashboard(uid: uid)
+
+        parse_json(response.body)
+      end
+
+      def fetch_datasource(dashboard)
+        name = DatasourceNameParser.new(grafana_url, dashboard).parse
+        raise DashboardProcessingError.new('Datasource name not found') unless name
+
+        response = client.get_datasource(name: name)
+
+        parse_json(response.body)
+      end
+
+      def grafana_url
+        params[:grafana_url]
+      end
+
+      def client
+        project.grafana_integration&.client
+      end
+
+      def allowed?
+        Ability.allowed?(current_user, :read_project, project)
+      end
+
+      def sequence
+        SEQUENCE
+      end
+
+      def parse_json(json)
+        JSON.parse(json, symbolize_names: true)
+      rescue JSON::ParserError
+        raise DashboardProcessingError.new('Grafana response contains invalid json')
+      end
+    end
+
+    # Identifies the uid of the dashboard based on url format
+    class GrafanaUidParser
+      def initialize(grafana_url, project)
+        @grafana_url, @project = grafana_url, project
+      end
+
+      def parse
+        @grafana_url.match(uid_regex) { |m| m.named_captures['uid'] }
+      end
+
+      private
+
+      # URLs are expected to look like https://domain.com/d/:uid/other/stuff
+      def uid_regex
+        base_url = @project.grafana_integration.grafana_url.chomp('/')
+
+        %r{(#{Regexp.escape(base_url)}\/d\/(?<uid>\w+)\/)}x
+      end
+    end
+
+    # Identifies the name of the datasource for a dashboard
+    # based on the panelId query parameter found in the url
+    class DatasourceNameParser
+      def initialize(grafana_url, grafana_dashboard)
+        @grafana_url, @grafana_dashboard = grafana_url, grafana_dashboard
+      end
+
+      def parse
+        @grafana_dashboard[:dashboard][:panels]
+          .find { |panel| panel[:id].to_s == query_params[:panelId] }
+          .try(:[], :datasource)
+      end
+
+      private
+
+      def query_params
+        Gitlab::Metrics::Dashboard::Url.parse_query(@grafana_url)
+      end
+    end
+  end
+end

app/views/admin/sessions/_new_base.html.haml

@@ -4,4 +4,4 @@
     = password_field_tag :password, nil, class: 'form-control', required: true, title: _('This field is required.'), data: { qa_selector: 'password_field' }
   .submit-container.move-submit-down
-    = submit_tag _('Enter admin mode'), class: 'btn btn-success', data: { qa_selector: 'sign_in_button' }
+    = submit_tag _('Enter Admin Mode'), class: 'btn btn-success', data: { qa_selector: 'sign_in_button' }

app/views/admin/sessions/_tabs_normal.html.haml

 %ul.nav-links.new-session-tabs.nav-tabs.nav{ role: 'tablist' }
   %li.nav-item{ role: 'presentation' }
-    %a.nav-link.active{ href: '#login-pane', data: { toggle: 'tab', qa_selector: 'sign_in_tab' }, role: 'tab' }= _('Enter admin mode')
+    %a.nav-link.active{ href: '#login-pane', data: { toggle: 'tab', qa_selector: 'sign_in_tab' }, role: 'tab' }= _('Enter Admin Mode')

app/views/admin/sessions/new.html.haml

 - @hide_breadcrumbs = true
-- page_title _('Enter admin mode')
+- page_title _('Enter Admin Mode')
 .row.justify-content-center
   .col-6.new-session-forms-container

app/views/layouts/nav/_dashboard.html.haml

@@ -55,15 +55,15 @@
             = nav_link(controller: 'admin/dashboard') do
               = link_to admin_root_path, class: 'admin-icon qa-admin-area-link d-xl-none' do
                 = _('Admin Area')
-            - if Feature.enabled?(:user_mode_in_session)
-              - if header_link?(:admin_mode)
-                = nav_link(controller: 'admin/sessions') do
-                  = link_to destroy_admin_session_path, class: 'd-lg-none lock-open-icon' do
-                    = _('Leave admin mode')
-              - elsif current_user.admin?
-                = nav_link(controller: 'admin/sessions') do
-                  = link_to new_admin_session_path, class: 'd-lg-none lock-icon' do
-                    = _('Enter admin mode')
+          - if Feature.enabled?(:user_mode_in_session)
+            - if header_link?(:admin_mode)
+              = nav_link(controller: 'admin/sessions') do
+                = link_to destroy_admin_session_path, class: 'd-lg-none lock-open-icon' do
+                  = _('Leave Admin Mode')
+            - elsif current_user.admin?
+              = nav_link(controller: 'admin/sessions') do
+                = link_to new_admin_session_path, class: 'd-lg-none lock-icon' do
+                  = _('Enter Admin Mode')
           - if Gitlab::Sherlock.enabled?
             %li
               = link_to sherlock_transactions_path, class: 'admin-icon' do
@@ -74,6 +74,15 @@
       = link_to admin_root_path, class: 'admin-icon qa-admin-area-link', title: _('Admin Area'), aria: { label: _('Admin Area') }, data: {toggle: 'tooltip', placement: 'bottom', container: 'body'} do
         = sprite_icon('admin', size: 18)
+  - if Feature.enabled?(:user_mode_in_session)
+    - if header_link?(:admin_mode)
+      = nav_link(controller: 'admin/sessions', html_options: { class: "d-none d-lg-block d-xl-block"}) do
+        = link_to destroy_admin_session_path, title: _('Leave Admin Mode'), aria: { label: _('Leave Admin Mode') }, data: { toggle: 'tooltip', placement: 'bottom', container: 'body' } do
+          = sprite_icon('lock-open', size: 18)
+    - elsif current_user.admin?
+      = nav_link(controller: 'admin/sessions', html_options: { class: "d-none d-lg-block d-xl-block"}) do
+        = link_to new_admin_session_path, title: _('Enter Admin Mode'), aria: { label: _('Enter Admin Mode') }, data: { toggle: 'tooltip', placement: 'bottom', container: 'body' } do
+          = sprite_icon('lock', size: 18)
   -# Shortcut to Dashboard > Projects
   - if dashboard_nav_link?(:projects)

changelogs/unreleased/34299-enable-color-chip-asciidoc.yml

+---
+title: Enable the color chip in AsciiDoc documents
+merge_request: 18723
+author:
+type: added

changelogs/unreleased/34320-error-when-uploading-a-few-designs-in-a-row.yml

+---
+title: Resolve Error when uploading a few designs in a row
+merge_request: 18811
+author:
+type: fixed

changelogs/unreleased/fix-admin-mode-ui-buttons-missing-on-small-screens.yml

+---
+title: Fix missing admin mode UI buttons on bigger screen sizes
+merge_request: 18585
+author: Diego Louzán
+type: fixed

lib/banzai/pipeline/ascii_doc_pipeline.rb

@@ -10,6 +10,7 @@ module Banzai
           Filter::SyntaxHighlightFilter,
           Filter::ExternalLinkFilter,
           Filter::PlantumlFilter,
+          Filter::ColorFilter,
           Filter::AsciiDocPostProcessingFilter
         ]
       end

lib/gitlab/metrics/dashboard/errors.rb

@@ -9,6 +9,7 @@ module Gitlab
       module Errors
         DashboardProcessingError = Class.new(StandardError)
         PanelNotFoundError = Class.new(StandardError)
+        MissingIntegrationError = Class.new(StandardError)
         LayoutError = Class.new(DashboardProcessingError)
         MissingQueryError = Class.new(DashboardProcessingError)
@@ -22,6 +23,10 @@ module Gitlab
             error("#{dashboard_path} could not be found.", :not_found)
           when PanelNotFoundError
             error(error.message, :not_found)
+          when ::Grafana::Client::Error
+            error(error.message, :service_unavailable)
+          when MissingIntegrationError
+            error('Proxy support for this API is not available currently', :bad_request)
           else
             raise error
           end

lib/gitlab/metrics/dashboard/processor.rb

@@ -17,7 +17,10 @@ module Gitlab
         # Returns a new dashboard hash with the results of
         # running transforms on the dashboard.
+        # @return [Hash, nil]
         def process
+          return unless @dashboard
+
           @dashboard.deep_symbolize_keys.tap do |dashboard|
             @sequence.each do |stage|
               stage.new(@project, dashboard, @params).transform!

lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class GrafanaFormatter < BaseStage
+          include Gitlab::Utils::StrongMemoize
+
+          CHART_TYPE = 'area-chart'
+          PROXY_PATH = 'api/v1/query_range'
+
+          # Reformats the specified panel in the Gitlab
+          # dashboard-yml format
+          def transform!
+            InputFormatValidator.new(
+              grafana_dashboard,
+              datasource,
+              panel,
+              query_params
+            ).validate!
+
+            new_dashboard = formatted_dashboard
+
+            dashboard.clear
+            dashboard.merge!(new_dashboard)
+          end
+
+          private
+
+          def formatted_dashboard
+            { panel_groups: [{ panels: [formatted_panel] }] }
+          end
+
+          def formatted_panel
+            {
+              title:   panel[:title],
+              type:    CHART_TYPE,
+              y_label: '', # Grafana panels do not include a Y-Axis label
+              metrics: panel[:targets].map.with_index do |target, idx|
+                formatted_metric(target, idx)
+              end
+            }
+          end
+
+          def formatted_metric(metric, idx)
+            {
+              id:                       "#{metric[:legendFormat]}_#{idx}",
+              query_range:              format_query(metric),
+              label:                    replace_variables(metric[:legendFormat]),
+              prometheus_endpoint_path: prometheus_endpoint_for_metric(metric)
+            }.compact
+          end
+
+          # Panel specified by the url from the Grafana dashboard
+          def panel
+            strong_memoize(:panel) do
+              grafana_dashboard[:dashboard][:panels].find do |panel|
+                panel[:id].to_s == query_params[:panelId]
+              end
+            end
+          end
+
+          # Grafana url query parameters. Includes information
+          # on which panel to select and time range.
+          def query_params
+            strong_memoize(:query_params) do
+              Gitlab::Metrics::Dashboard::Url.parse_query(grafana_url)
+            end
+          end
+
+          # Endpoint which will return prometheus metric data
+          # for the metric
+          def prometheus_endpoint_for_metric(metric)
+            Gitlab::Routing.url_helpers.project_grafana_api_path(
+              project,
+              datasource_id: datasource[:id],
+              proxy_path: PROXY_PATH,
+              query: format_query(metric)
+            )
+          end
+
+          # Reformats query for compatibility with prometheus api.
+          def format_query(metric)
+            expression = remove_new_lines(metric[:expr])
+            expression = replace_variables(expression)
+            expression = replace_global_variables(expression, metric)
+
+            expression
+          end
+
+          # Accomodates instance-defined Grafana variables.
+          # These are variables defined by users, and values
+          # must be provided in the query parameters.
+          def replace_variables(expression)
+            return expression unless grafana_dashboard[:dashboard][:templating]
+
+            grafana_dashboard[:dashboard][:templating][:list]
+              .sort_by { |variable| variable[:name].length }
+              .each do |variable|
+                variable_value = query_params[:"var-#{variable[:name]}"]
+
+                expression = expression.gsub("$#{variable[:name]}", variable_value)
+                expression = expression.gsub("[[#{variable[:name]}]]", variable_value)
+                expression = expression.gsub("{{#{variable[:name]}}}", variable_value)
+              end
+
+            expression
+          end
+
+          # Replaces Grafana global built-in variables with values.
+          # Only $__interval and $__from and $__to are supported.
+          #
+          # See https://grafana.com/docs/reference/templating/#global-built-in-variables
+          def replace_global_variables(expression, metric)
+            expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval]
+            expression = expression.gsub('$__from', query_params[:from])
+            expression = expression.gsub('$__to', query_params[:to])
+
+            expression
+          end
+
+          # Removes new lines from expression.
+          def remove_new_lines(expression)
+            expression.gsub(/\R+/, '')
+          end
+
+          # Grafana datasource object corresponding to the
+          # specified dashboard
+          def datasource
+            params[:datasource]
+          end
+
+          # The specified Grafana dashboard
+          def grafana_dashboard
+            params[:grafana_dashboard]
+          end
+
+          # The URL specifying which Grafana panel to embed
+          def grafana_url
+            params[:grafana_url]
+          end
+        end
+
+        class InputFormatValidator
+          include ::Gitlab::Metrics::Dashboard::Errors
+
+          attr_reader :grafana_dashboard, :datasource, :panel, :query_params
+
+          UNSUPPORTED_GRAFANA_GLOBAL_VARS = %w(
+            $__interval_ms
+            $__timeFilter
+            $__name
+            $timeFilter
+            $interval
+          ).freeze
+
+          def initialize(grafana_dashboard, datasource, panel, query_params)
+            @grafana_dashboard = grafana_dashboard
+            @datasource = datasource
+            @panel = panel
+            @query_params = query_params
+          end
+
+          def validate!
+            validate_query_params!
+            validate_datasource!
+            validate_panel_type!
+            validate_variable_definitions!
+            validate_global_variables!
+          end
+
+          private
+
+          def validate_datasource!
+            return if datasource[:access] == 'proxy' && datasource[:type] == 'prometheus'
+
+            raise_error 'Only Prometheus datasources with proxy access in Grafana are supported.'
+          end
+
+          def validate_query_params!
+            return if [:panelId, :from, :to].all? { |param| query_params.include?(param) }
+
+            raise_error 'Grafana query parameters must include panelId, from, and to.'
+          end
+
+          def validate_panel_type!
+            return if panel[:type] == 'graph' && panel[:lines]
+
+            raise_error 'Panel type must be a line graph.'
+          end
+
+          def validate_variable_definitions!
+            return unless grafana_dashboard[:dashboard][:templating]
+
+            return if grafana_dashboard[:dashboard][:templating][:list].all? do |variable|
+              query_params[:"var-#{variable[:name]}"].present?
+            end
+
+            raise_error 'All Grafana variables must be defined in the query parameters.'
+          end
+
+          def validate_global_variables!
+            return unless panel_contains_unsupported_vars?
+
+            raise_error 'Prometheus must not include'
+          end
+
+          def panel_contains_unsupported_vars?
+            panel[:targets].any? do |target|
+              UNSUPPORTED_GRAFANA_GLOBAL_VARS.any? do |variable|
+                target[:expr].include?(variable)
+              end
+            end
+          end
+
+          def raise_error(message)
+            raise DashboardProcessingError.new(message)
+          end
+        end
+      end
+    end
+  end
+end

lib/grafana/client.rb

@@ -11,6 +11,18 @@ module Grafana
       @token = token
     end
+    # @param uid [String] Unique identifier for a Grafana dashboard
+    def get_dashboard(uid:)
+      http_get("#{@api_url}/api/dashboards/uid/#{uid}")
+    end
+
+    # @param name [String] Unique identifier for a Grafana datasource
+    def get_datasource(name:)
+      # CGI#escape formats strings such that the Grafana endpoint
+      # will not recognize the dashboard name. Preferring URI#escape.
+      http_get("#{@api_url}/api/datasources/name/#{URI.escape(name)}") # rubocop:disable Lint/UriEscapeUnescape
+    end
+
     # @param datasource_id [String] Grafana ID for the datasource
     # @param proxy_path [String] Path to proxy - ex) 'api/v1/query_range'
     def proxy_datasource(datasource_id:, proxy_path:, query: {})
@@ -57,7 +69,7 @@ module Grafana
     def handle_response(response)
       return response if response.code == 200
-      raise_error "Grafana response status code: #{response.code}"
+      raise_error "Grafana response status code: #{response.code}, Message: #{response.body}"
     end
     def raise_error(message)

locale/gitlab.pot

@@ -6099,13 +6099,13 @@ msgstr ""
 msgid "Ensure your %{linkStart}environment is part of the deploy stage%{linkEnd} of your CI pipeline to track deployments to your cluster."
 msgstr ""
-msgid "Enter IP address range"
+msgid "Enter Admin Mode"
 msgstr ""
-msgid "Enter a number"
+msgid "Enter IP address range"
 msgstr ""
-msgid "Enter admin mode"
+msgid "Enter a number"
 msgstr ""
 msgid "Enter at least three characters to search"
@@ -9680,7 +9680,7 @@ msgstr ""
 msgid "Leave"
 msgstr ""
-msgid "Leave admin mode"
+msgid "Leave Admin Mode"
 msgstr ""
 msgid "Leave edit mode? All unsaved changes will be lost."

spec/factories/grafana_integrations.rb

@@ -3,7 +3,7 @@
 FactoryBot.define do
   factory :grafana_integration, class: GrafanaIntegration do
     project
-    grafana_url { 'https://grafana.com' }
+    grafana_url { 'https://grafana.example.com' }
     token { SecureRandom.hex(10) }
   end
 end

spec/features/admin/admin_settings_spec.rb

@@ -5,6 +5,7 @@ require 'spec_helper'
 describe 'Admin updates settings', :clean_gitlab_redis_shared_state, :do_not_mock_admin_mode do
   include StubENV
   include TermsHelper
+  include MobileHelpers
   let(:admin) { create(:admin) }
@@ -450,6 +451,32 @@ describe 'Admin updates settings', :clean_gitlab_redis_shared_state, :do_not_moc
           expect(page).to have_link(text: 'Support', href: new_support_url)
         end
       end
+
+      it 'Shows admin dashboard links on bigger screen' do
+        visit root_dashboard_path
+
+        page.within '.navbar' do
+          expect(page).to have_link(text: 'Admin Area', href: admin_root_path, visible: true)
+          expect(page).to have_link(text: 'Leave Admin Mode', href: destroy_admin_session_path, visible: true)
+        end
+      end
+
+      it 'Relocates admin dashboard links to dropdown list on smaller screen', :js do
+        resize_screen_xs
+        visit root_dashboard_path
+
+        page.within '.navbar' do
+          expect(page).not_to have_link(text: 'Admin Area', href: admin_root_path, visible: true)
+          expect(page).not_to have_link(text: 'Leave Admin Mode', href: destroy_admin_session_path, visible: true)
+        end
+
+        find('.header-more').click
+
+        page.within '.navbar' do
+          expect(page).to have_link(text: 'Admin Area', href: admin_root_path, visible: true)
+          expect(page).to have_link(text: 'Leave Admin Mode', href: destroy_admin_session_path, visible: true)
+        end
+      end
     end
     context 'when in admin_mode' do
@@ -462,7 +489,7 @@ describe 'Admin updates settings', :clean_gitlab_redis_shared_state, :do_not_moc
       it 'can leave admin mode' do
         page.within('.navbar-sub-nav') do
           # Select first, link is also included in mobile view list
-          click_on 'Leave admin mode', match: :first
+          click_on 'Leave Admin Mode', match: :first
           expect(page).to have_link(href: new_admin_session_path)
         end
@@ -481,7 +508,7 @@ describe 'Admin updates settings', :clean_gitlab_redis_shared_state, :do_not_moc
       before do
         page.within('.navbar-sub-nav') do
           # Select first, link is also included in mobile view list
-          click_on 'Leave admin mode', match: :first
+          click_on 'Leave Admin Mode', match: :first
         end
       end