Add latest changes from gitlab-org/gitlab@master

CHANGELOG-EE.md

 Please view this file on the master branch, on stable branches it's out of date.
+## 12.8.6 (2020-03-11)
+
+- No changes.
+
 ## 12.8.5
 - No changes.

app/controllers/concerns/confirm_email_warning.rb

@@ -10,7 +10,7 @@ module ConfirmEmailWarning
   protected
   def show_confirm_warning?
-    html_request? && request.get?
+    html_request? && request.get? && Feature.enabled?(:soft_email_confirmation)
   end
   def set_confirm_warning

app/controllers/confirmations_controller.rb

@@ -11,6 +11,8 @@ class ConfirmationsController < Devise::ConfirmationsController
   protected
   def after_resending_confirmation_instructions_path_for(resource)
+    return users_almost_there_path unless Feature.enabled?(:soft_email_confirmation)
+
     stored_location_for(resource) || dashboard_projects_path
   end

app/controllers/registrations_controller.rb

@@ -54,7 +54,7 @@ class RegistrationsController < Devise::RegistrationsController
   def welcome
     return redirect_to new_user_registration_path unless current_user
-    return redirect_to stored_location_or_dashboard(current_user) if current_user.role.present? && !current_user.setup_for_company.nil?
+    return redirect_to path_for_signed_in_user(current_user) if current_user.role.present? && !current_user.setup_for_company.nil?
   end
   def update_registration
@@ -64,7 +64,7 @@ class RegistrationsController < Devise::RegistrationsController
     if result[:status] == :success
       track_experiment_event(:signup_flow, 'end') # We want this event to be tracked when the user is _in_ the experimental group
       set_flash_message! :notice, :signed_up
-      redirect_to stored_location_or_dashboard(current_user)
+      redirect_to path_for_signed_in_user(current_user)
     else
       render :welcome
     end
@@ -111,14 +111,12 @@ class RegistrationsController < Devise::RegistrationsController
     return users_sign_up_welcome_path if experiment_enabled?(:signup_flow)
-    stored_location_or_dashboard(user)
+    path_for_signed_in_user(user)
   end
   def after_inactive_sign_up_path_for(resource)
-    # With the current `allow_unconfirmed_access_for` Devise setting in config/initializers/8_devise.rb,
-    # this method is never called. Leaving this here in case that value is set to 0.
     Gitlab::AppLogger.info(user_created_message)
-    users_almost_there_path
+    Feature.enabled?(:soft_email_confirmation) ? dashboard_projects_path : users_almost_there_path
   end
   private
@@ -180,8 +178,20 @@ class RegistrationsController < Devise::RegistrationsController
     Gitlab::Utils.to_boolean(params[:terms_opt_in])
   end
-  def stored_location_or_dashboard(user)
-    stored_location_for(user) || dashboard_projects_path
+  def path_for_signed_in_user(user)
+    if requires_confirmation?(user)
+      users_almost_there_path
+    else
+      stored_location_for(user) || dashboard_projects_path
+    end
+  end
+
+  def requires_confirmation?(user)
+    return false if user.confirmed?
+    return false if Feature.enabled?(:soft_email_confirmation)
+    return false if experiment_enabled?(:signup_flow)
+
+    true
   end
   def load_recaptcha

app/models/commit.rb

@@ -242,14 +242,6 @@ class Commit
     data
   end
-  # Discover issues should be closed when this commit is pushed to a project's
-  # default branch.
-  def closes_issues(current_user = self.committer)
-    return unless repository.repo_type.project?
-
-    Gitlab::ClosingIssueExtractor.new(project, current_user).closed_by_message(safe_message)
-  end
-
   def lazy_author
     BatchLoader.for(author_email.downcase).batch do |emails, loader|
       users = User.by_any_email(emails, confirmed: true).includes(:emails)
@@ -299,14 +291,6 @@ class Commit
     notes.includes(:author, :award_emoji)
   end
-  def merge_requests
-    strong_memoize(:merge_requests) do
-      next MergeRequest.none unless repository.repo_type.project? && project
-
-      project.merge_requests.by_commit_sha(sha)
-    end
-  end
-
   def method_missing(method, *args, &block)
     @raw.__send__(method, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
   end

app/models/merge_request.rb

@@ -257,6 +257,10 @@ class MergeRequest < ApplicationRecord
     with_state(:opened).where(auto_merge_enabled: true)
   end
+  scope :including_metrics, -> do
+    includes(:metrics)
+  end
+
   ignore_column :state, remove_with: '12.7', remove_after: '2019-12-22'
   after_save :keep_around_commit, unless: :importing?

app/models/note.rb

@@ -290,6 +290,19 @@ class Note < ApplicationRecord
     @commit ||= project.commit(commit_id) if commit_id.present?
   end
+  # Notes on merge requests and commits can be traced back to one or several
+  # MRs. This method returns a relation if the note is for one of these types,
+  # or nil if it is a note on some other object.
+  def merge_requests
+    if for_commit?
+      project.merge_requests.by_commit_sha(commit_id)
+    elsif for_merge_request?
+      MergeRequest.id_in(noteable_id)
+    else
+      nil
+    end
+  end
+
   # override to return commits, which are not active record
   def noteable
     return commit if for_commit?

app/models/project.rb

@@ -1787,8 +1787,10 @@ class Project < ApplicationRecord
   # rubocop:enable Gitlab/RailsLogger
   def after_import
-    repository.after_import
-    wiki.repository.after_import
+    repository.expire_content_cache
+    wiki.repository.expire_content_cache
+
+    DetectRepositoryLanguagesWorker.perform_async(id)
     # The import assigns iid values on its own, e.g. by re-using GitHub ids.
     # Flush existing InternalId records for this project for consistency reasons.

app/models/repository.rb

@@ -437,15 +437,6 @@ class Repository
     expire_all_method_caches
   end
-  # Runs code after a repository has been forked/imported.
-  def after_import
-    expire_content_cache
-
-    return unless repo_type.project?
-
-    DetectRepositoryLanguagesWorker.perform_async(project.id)
-  end
-
   # Runs code after a new commit has been pushed.
   def after_push_commit(branch_name)
     expire_statistics_caches

app/models/user.rb

@@ -1683,6 +1683,13 @@ class User < ApplicationRecord
     super
   end
+  # override from Devise::Confirmable
+  def confirmation_period_valid?
+    return false if Feature.disabled?(:soft_email_confirmation)
+
+    super
+  end
+
   private
   def default_private_profile_to_false

app/services/issues/close_service.rb

@@ -52,12 +52,12 @@ module Issues
     end
     def store_first_mentioned_in_commit_at(issue, merge_request)
-      return unless Feature.enabled?(:store_first_mentioned_in_commit_on_issue_close, issue.project)
+      return unless Feature.enabled?(:store_first_mentioned_in_commit_on_issue_close, issue.project, default_enabled: true)
       metrics = issue.metrics
       return if metrics.nil? || metrics.first_mentioned_in_commit_at
-      first_commit_timestamp = merge_request.commits(limit: 1).first&.date
+      first_commit_timestamp = merge_request.commits(limit: 1).first.try(:authored_date)
       return unless first_commit_timestamp
       metrics.update!(first_mentioned_in_commit_at: first_commit_timestamp)

app/workers/process_commit_worker.rb

@@ -19,13 +19,12 @@ class ProcessCommitWorker # rubocop:disable Scalability/IdempotentWorker
   # commit_hash - Hash containing commit details to use for constructing a
   #               Commit object without having to use the Git repository.
   # default - The data was pushed to the default branch.
-  # rubocop: disable CodeReuse/ActiveRecord
   def perform(project_id, user_id, commit_hash, default = false)
-    project = Project.find_by(id: project_id)
+    project = Project.id_in(project_id).first
     return unless project
-    user = User.find_by(id: user_id)
+    user = User.id_in(user_id).first
     return unless user
@@ -35,12 +34,11 @@ class ProcessCommitWorker # rubocop:disable Scalability/IdempotentWorker
     process_commit_message(project, commit, user, author, default)
     update_issue_metrics(commit, author)
   end
-  # rubocop: enable CodeReuse/ActiveRecord
   def process_commit_message(project, commit, user, author, default = false)
     # Ignore closing references from GitLab-generated commit messages.
     find_closing_issues = default && !commit.merged_merge_request?(user)
-    closed_issues = find_closing_issues ? commit.closes_issues(user) : []
+    closed_issues = find_closing_issues ? issues_to_close(project, commit, user) : []
     close_issues(project, user, author, commit, closed_issues) if closed_issues.any?
     commit.create_cross_references!(author, closed_issues)
@@ -56,6 +54,12 @@ class ProcessCommitWorker # rubocop:disable Scalability/IdempotentWorker
     end
   end
+  def issues_to_close(project, commit, user)
+    Gitlab::ClosingIssueExtractor
+      .new(project, user)
+      .closed_by_message(commit.safe_message)
+  end
+
   def update_issue_metrics(commit, author)
     mentioned_issues = commit.all_references(author).issues

bin/sidekiq-cluster

+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'optparse'
+require_relative '../lib/gitlab'
+require_relative '../lib/gitlab/utils'
+require_relative '../lib/gitlab/sidekiq_config/cli_methods'
+require_relative '../lib/gitlab/sidekiq_cluster'
+require_relative '../lib/gitlab/sidekiq_cluster/cli'
+
+Thread.abort_on_exception = true
+
+cli = Gitlab::SidekiqCluster::CLI.new
+
+begin
+  cli.run
+rescue Gitlab::SidekiqCluster::CLI::CommandError => error
+  abort error.message
+end

changelogs/unreleased/202094-enable-ff-by-default.yml

+---
+title: Store first commit's authored_date for value stream calculation on merge
+merge_request: 26885
+author:
+type: changed

changelogs/unreleased/create_single_temporary_index_for_notes_mentions.yml

+---
+title: Replace several temporary indexes with a single one to save time when running mentions migration
+merge_request:
+author:
+type: performance

changelogs/unreleased/osw-move-sidekiq-cluster-to-core.yml

+---
+title: Move sidekiq-cluster script to Core
+merge_request: 26703
+author:
+type: other

db/post_migrate/20200128132510_add_temporary_index_for_notes_with_mentions.rb

+# frozen_string_literal: true
+
+class AddTemporaryIndexForNotesWithMentions < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  INDEX_CONDITION = "note LIKE '%@%'::text"
+  INDEX_NAME = 'note_mentions_temp_index'
+
+  EPIC_MENTIONS_INDEX_NAME = 'epic_mentions_temp_index'
+  DESIGN_MENTIONS_INDEX_NAME = 'design_mentions_temp_index'
+
+  def up
+    # create temporary index for notes with mentions, may take well over 1h
+    add_concurrent_index(:notes, [:id, :noteable_type], where: INDEX_CONDITION, name: INDEX_NAME)
+
+    # cleanup previous temporary indexes, as we'll be usig the single one
+    remove_concurrent_index(:notes, :id, name: EPIC_MENTIONS_INDEX_NAME)
+    remove_concurrent_index(:notes, :id, name: DESIGN_MENTIONS_INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index(:notes, :id, name: INDEX_NAME)
+
+    add_concurrent_index(:notes, :id, where: "#{INDEX_CONDITION} AND noteable_type='Epic'", name: EPIC_MENTIONS_INDEX_NAME)
+    add_concurrent_index(:notes, :id, where: "#{INDEX_CONDITION} AND noteable_type='DesignManagement::Design'", name: DESIGN_MENTIONS_INDEX_NAME)
+  end
+end

db/schema.rb

@@ -2840,8 +2840,7 @@ ActiveRecord::Schema.define(version: 2020_03_10_135823) do
     t.index ["commit_id"], name: "index_notes_on_commit_id"
     t.index ["created_at"], name: "index_notes_on_created_at"
     t.index ["discussion_id"], name: "index_notes_on_discussion_id"
-    t.index ["id"], name: "design_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'DesignManagement::Design'::text))"
-    t.index ["id"], name: "epic_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'Epic'::text))"
+    t.index ["id", "noteable_type"], name: "note_mentions_temp_index", where: "(note ~~ '%@%'::text)"
     t.index ["line_code"], name: "index_notes_on_line_code"
     t.index ["note"], name: "index_notes_on_note_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["note"], name: "tmp_idx_on_promoted_notes", where: "(((noteable_type)::text = 'Issue'::text) AND (system IS TRUE) AND (note ~~ 'promoted to epic%'::text))"

doc/administration/geo/disaster_recovery/index.md

@@ -139,6 +139,8 @@ do this manually.
    sudo gitlab-pg-ctl promote
    ```
+   In GitLab 12.8 and earlier, see [Message: "sudo: gitlab-pg-ctl: command not found"](../replication/troubleshooting.md#message-sudo-gitlab-pg-ctl-command-not-found).
+
 1. Edit `/etc/gitlab/gitlab.rb` on every machine in the **secondary** to
    reflect its new status as **primary** by removing any lines that enabled the
    `geo_secondary_role`:

doc/administration/geo/replication/troubleshooting.md

@@ -514,6 +514,27 @@ or `gitlab-ctl promote-to-primary-node`, either:
   bug](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22021) was
   fixed.
+### Message: "sudo: gitlab-pg-ctl: command not found"
+
+When
+[promoting a **secondary** node with HA](../disaster_recovery/index.md#promoting-a-secondary-node-with-ha),
+you need to run the `gitlab-pg-ctl` command to promote the PostgreSQL
+read-replica database.
+
+In GitLab 12.8 and earlier, this command will fail with the message:
+
+```plaintext
+sudo: gitlab-pg-ctl: command not found
+```
+
+In this case, the workaround is to use the full path to the binary, for example:
+
+```shell
+sudo /opt/gitlab/embedded/bin/gitlab-pg-ctl promote
+```
+
+GitLab 12.9 and later are [unaffected by this error](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5147).
+
 ## Fixing Foreign Data Wrapper errors
 This section documents ways to fix potential Foreign Data Wrapper errors.