Merge remote-tracking branch 'dev/12-8-stable' into 12-8-stable

app/validators/addressable_url_validator.rb

@@ -23,7 +23,8 @@
 # protect against Server-side Request Forgery (SSRF), or check for the right port.
 #
 # Configuration options:
-# * <tt>message</tt> - A custom error message (default is: "must be a valid URL").
+# * <tt>message</tt> - A custom error message, used when the URL is blank. (default is: "must be a valid URL").
+# * <tt>blocked_message</tt> - A custom error message, used when the URL is blocked. Default: +'is blocked: %{exception_message}'+.
 # * <tt>schemes</tt> - Array of URI schemes. Default: +['http', 'https']+
 # * <tt>allow_localhost</tt> - Allow urls pointing to +localhost+. Default: +true+
 # * <tt>allow_local_network</tt> - Allow urls pointing to private network addresses. Default: +true+
@@ -59,7 +60,8 @@ class AddressableUrlValidator < ActiveModel::EachValidator
   }.freeze
   DEFAULT_OPTIONS = BLOCKER_VALIDATE_OPTIONS.merge({
-    message: 'must be a valid URL'
+    message: 'must be a valid URL',
+    blocked_message: 'is blocked: %{exception_message}'
   }).freeze
   def initialize(options)
@@ -80,7 +82,7 @@ class AddressableUrlValidator < ActiveModel::EachValidator
     Gitlab::UrlBlocker.validate!(value, blocker_args)
   rescue Gitlab::UrlBlocker::BlockedUrlError => e
-    record.errors.add(attribute, "is blocked: #{e.message}")
+    record.errors.add(attribute, options.fetch(:blocked_message) % { exception_message: e.message })
   end
   private

app/views/admin/application_settings/_grafana.html.haml

-= form_for @application_setting, url: admin_application_settings_path(anchor: 'js-grafana-settings'), html: { class: 'fieldset-form' } do |f|
+= form_for @application_setting, url: metrics_and_profiling_admin_application_settings_path(anchor: 'js-grafana-settings'), html: { class: 'fieldset-form' } do |f|
   = form_errors(@application_setting)
   %fieldset

app/views/layouts/nav/sidebar/_admin.html.haml

@@ -83,7 +83,7 @@
                 = _('Requests Profiles')
           - if Gitlab::CurrentSettings.current_application_settings.grafana_enabled?
             = nav_link do
-              = link_to Gitlab::CurrentSettings.current_application_settings.grafana_url, target: '_blank', title: _('Metrics Dashboard') do
+              = link_to Gitlab::CurrentSettings.current_application_settings.grafana_url, target: '_blank', title: _('Metrics Dashboard'), rel: 'noopener noreferrer' do
                 %span
                   = _('Metrics Dashboard')
           = render_if_exists 'layouts/nav/ee/admin/new_monitoring_sidebar'

app/views/shared/issuable/form/_branch_chooser.html.haml

@@ -8,7 +8,9 @@
 .form-group.row.d-flex.gl-pl-3.gl-pr-3.branch-selector
   .align-self-center
-    %span= s_('From %{source_title} into').html_safe % { source_title: "<code>#{source_title}</code>".html_safe }
+    %span
+      = _('From <code>%{source_title}</code> into').html_safe % { source_title: source_title }
+
     - if issuable.new_record?
       %code= target_title
       &nbsp;

config/initializers/8_devise.rb

@@ -80,8 +80,16 @@ Devise.setup do |config|
   # When allow_unconfirmed_access_for is zero, the user won't be able to sign in without confirming.
   # You can use this to let your user access some features of your application
   # without confirming the account, but blocking it after a certain period
-  # (ie 2 days).
-  config.allow_unconfirmed_access_for = 30.days
+  # (e.g. 3 days).
+  config.allow_unconfirmed_access_for = 3.days
+
+  # A period that the user is allowed to confirm their account before their
+  # token becomes invalid. For example, if set to 1.day, the user can confirm
+  # their account within 1 days after the mail was sent, but on the second day
+  # their account can't be confirmed with the token any more.
+  # Default is nil, meaning there is no restriction on how long a user can take
+  # before confirming their account.
+  config.confirm_within = 1.day
   # Defines which key will be used when confirming an account
   # config.confirmation_keys = [ :email ]

db/migrate/20200214085940_clean_grafana_url.rb

+# frozen_string_literal: true
+
+class CleanGrafanaUrl < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    execute(
+      <<-SQL
+      UPDATE
+        application_settings
+      SET
+        grafana_url = default
+      WHERE
+        position('javascript:' IN btrim(application_settings.grafana_url)) = 1
+      SQL
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20200204113224_schedule_recalculate_project_authorizations_second_run.rb

+# frozen_string_literal: true
+
+class ScheduleRecalculateProjectAuthorizationsSecondRun < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'RecalculateProjectAuthorizationsWithMinMaxUserId'
+  BATCH_SIZE = 2_500
+  DELAY_INTERVAL = 2.minutes.to_i
+
+  disable_ddl_transaction!
+
+  class User < ActiveRecord::Base
+    include ::EachBatch
+
+    self.table_name = 'users'
+  end
+
+  def up
+    say "Scheduling #{MIGRATION} jobs"
+
+    User.each_batch(of: BATCH_SIZE) do |batch, index|
+      delay = index * DELAY_INTERVAL
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+      BackgroundMigrationWorker.perform_in(delay, MIGRATION, range)
+    end
+  end
+
+  def down
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2020_02_13_220211) do
+ActiveRecord::Schema.define(version: 2020_02_14_085940) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"

doc/administration/monitoring/performance/grafana_configuration.md

@@ -117,8 +117,9 @@ If you have set up Grafana, you can enable a link to access it easily from the s
 1. Expand **Metrics - Grafana**.
 1. Check the "Enable access to Grafana" checkbox.
 1. If Grafana is enabled through Omnibus GitLab and on the same server,
-   leave "Grafana URL" unchanged. In any other case, enter the full URL
-   path of the Grafana instance.
+   leave **Grafana URL** unchanged. It should be `/-/grafana`.
+
+   In any other case, enter the full URL of the Grafana instance.
 1. Click **Save changes**.
 1. The new link will be available in the **Admin Area > Monitoring > Metrics Dashboard**.

doc/api/graphql/reference/gitlab_schema.graphql

@@ -1530,7 +1530,7 @@ type DiffRefs {
   """
   Merge base of the branch the comment was made on
   """
-  baseSha: String!
+  baseSha: String
   """
   SHA of the HEAD at the time the comment was made

doc/api/graphql/reference/gitlab_schema.json

@@ -8105,13 +8105,9 @@
               ],
               "type": {
-                "kind": "NON_NULL",
-                "name": null,
-                "ofType": {
-                  "kind": "SCALAR",
-                  "name": "String",
-                  "ofType": null
-                }
+                "kind": "SCALAR",
+                "name": "String",
+                "ofType": null
               },
               "isDeprecated": false,
               "deprecationReason": null

doc/api/graphql/reference/index.md

@@ -253,7 +253,7 @@ Autogenerated return type of DestroySnippet
 | Name  | Type  | Description |
 | ---   |  ---- | ----------  |
-| `baseSha` | String! | Merge base of the branch the comment was made on |
+| `baseSha` | String | Merge base of the branch the comment was made on |
 | `headSha` | String! | SHA of the HEAD at the time the comment was made |
 | `startSha` | String! | SHA of the branch being compared against |

lib/api/triggers.rb

@@ -4,6 +4,8 @@ module API
   class Triggers < Grape::API
     include PaginationParams
+    HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase
+
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
@@ -19,6 +21,8 @@ module API
       post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do
         Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42283')
+        forbidden! if gitlab_pipeline_hook_request?
+
         # validate variables
         params[:variables] = params[:variables].to_h
         unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) }
@@ -128,5 +132,11 @@ module API
         destroy_conditionally!(trigger)
       end
     end
+
+    helpers do
+      def gitlab_pipeline_hook_request?
+        request.get_header(HTTP_GITLAB_EVENT_HEADER) == WebHookService.hook_to_event(:pipeline_hooks)
+      end
+    end
   end
 end

lib/gitlab/asset_proxy.rb

+# frozen_string_literal: true
+
+# This is based on https://github.com/jch/html-pipeline/blob/v2.12.2/lib/html/pipeline/camo_filter.rb
+# and Banzai::Filter::AssetProxyFilter which we use to proxy images in Markdown
+
+module Gitlab
+  module AssetProxy
+    class << self
+      def proxy_url(url)
+        return url unless Gitlab.config.asset_proxy.enabled
+        return url if asset_host_whitelisted?(url)
+
+        "#{Gitlab.config.asset_proxy.url}/#{asset_url_hash(url)}/#{hexencode(url)}"
+      end
+
+      private
+
+      def asset_host_whitelisted?(url)
+        parsed_url = URI.parse(url)
+
+        Gitlab.config.asset_proxy.domain_regexp&.match?(parsed_url.host)
+      end
+
+      def asset_url_hash(url)
+        OpenSSL::HMAC.hexdigest('sha1', Gitlab.config.asset_proxy.secret_key, url)
+      end
+
+      def hexencode(str)
+        str.unpack1('H*')
+      end
+    end
+  end
+end

lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop:disable Style/Documentation
+    class RecalculateProjectAuthorizationsWithMinMaxUserId
+      def perform(min_user_id, max_user_id)
+        User.where(id: min_user_id..max_user_id).find_each do |user|
+          service = Users::RefreshAuthorizedProjectsService.new(
+            user,
+            incorrect_auth_found_callback:
+              ->(project_id, access_level) do
+                logger.info(message: 'Removing ProjectAuthorizations',
+                            user_id: user.id,
+                            project_id: project_id,
+                            access_level: access_level)
+              end,
+            missing_auth_found_callback:
+              ->(project_id, access_level) do
+                logger.info(message: 'Creating ProjectAuthorizations',
+                            user_id: user.id,
+                            project_id: project_id,
+                            access_level: access_level)
+              end
+          )
+
+          service.execute
+        end
+      end
+
+      private
+
+      def logger
+        @logger ||= Gitlab::BackgroundMigration::Logger.build
+      end
+    end
+  end
+end

lib/gitlab/dependency_linker/base_linker.rb

@@ -7,6 +7,8 @@ module Gitlab
       GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze
       REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze
+      include ActionView::Helpers::SanitizeHelper
+
       class_attribute :file_type
       def self.support?(blob_name)
@@ -62,7 +64,10 @@ module Gitlab
       end
       def link_tag(name, url)
-        %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}.html_safe
+        sanitize(
+          %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>},
+          attributes: %w[href rel target]
+        )
       end
       # Links package names based on regex.

lib/gitlab/project_authorizations.rb

@@ -62,6 +62,7 @@ module Gitlab
       cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte)
       members = Member.arel_table
       namespaces = Namespace.arel_table
+      group_group_links = GroupGroupLink.arel_table
       # Namespaces the user is a member of.
       cte << user.groups
@@ -69,7 +70,10 @@ module Gitlab
         .except(:order)
       # Namespaces shared with any of the group
-      cte << Group.select([namespaces[:id], 'group_group_links.group_access AS access_level'])
+      cte << Group.select([namespaces[:id],
+                           least(members[:access_level],
+                                 group_group_links[:group_access],
+                                 'access_level')])
                   .joins(join_group_group_links)
                   .joins(join_members_on_group_group_links)

lib/gitlab/user_access.rb

@@ -67,7 +67,13 @@ module Gitlab
       return false unless can_access_git?
       return false unless project
-      return false if !user.can?(:push_code, project) && !project.branch_allows_collaboration?(user, ref)
+      # Checking for an internal project to prevent an infinite loop:
+      # https://gitlab.com/gitlab-org/gitlab/issues/36805
+      if project.internal?
+        return false unless user.can?(:push_code, project)
+      else
+        return false if !user.can?(:push_code, project) && !project.branch_allows_collaboration?(user, ref)
+      end
       if protected?(ProtectedBranch, project, ref)
         protected_branch_accessible_to?(ref, action: :push)

lib/gitlab/utils.rb

@@ -136,5 +136,14 @@ module Gitlab
       IPAddr.new(str)
     rescue IPAddr::InvalidAddressError
     end
+
+    # Converts a string to an Addressable::URI object.
+    # If the string is not a valid URI, it returns nil.
+    # Param uri_string should be a String object.
+    # This method returns an Addressable::URI object or nil.
+    def parse_url(uri_string)
+      Addressable::URI.parse(uri_string)
+    rescue Addressable::URI::InvalidURIError, TypeError
+    end
   end
 end

locale/gitlab.pot

@@ -8208,6 +8208,9 @@ msgstr ""
 msgid "Failure"
 msgstr ""
+msgid "Fast-forward merge is not possible. Rebase the source branch onto %{targetBranch} to allow this merge request to be merged."
+msgstr ""
+
 msgid "Fast-forward merge is not possible. Rebase the source branch onto the target branch or merge target branch into source branch to allow this merge request to be merged."
 msgstr ""
@@ -8655,7 +8658,7 @@ msgstr ""
 msgid "From %{providerTitle}"
 msgstr ""
-msgid "From %{source_title} into"
+msgid "From <code>%{source_title}</code> into"
 msgstr ""
 msgid "From Bitbucket"