Add latest changes from gitlab-org/gitlab@master

app/models/merge_request_diff.rb

@@ -560,6 +560,10 @@ class MergeRequestDiff < ApplicationRecord
     opening_external_diff do
       collection = merge_request_diff_files
+      if options[:include_context_commits]
+        collection += merge_request.merge_request_context_commit_diff_files
+      end
+
       if paths = options[:paths]
         collection = collection.where('old_path IN (?) OR new_path IN (?)', paths, paths)
       end

app/models/merge_request_diff_commit.rb

@@ -2,6 +2,7 @@
 class MergeRequestDiffCommit < ApplicationRecord
   include ShaAttribute
+  include CachedCommit
   belongs_to :merge_request_diff
@@ -9,8 +10,6 @@ class MergeRequestDiffCommit < ApplicationRecord
   alias_attribute :id, :sha
   def self.create_bulk(merge_request_diff_id, commits)
-    sha_attribute = Gitlab::Database::ShaAttribute.new
-
     rows = commits.map.with_index do |commit, index|
       # See #parent_ids.
       commit_hash = commit.to_hash.except(:parent_ids)
@@ -19,7 +18,7 @@ class MergeRequestDiffCommit < ApplicationRecord
       commit_hash.merge(
         merge_request_diff_id: merge_request_diff_id,
         relative_order: index,
-        sha: sha_attribute.serialize(sha), # rubocop:disable Cop/ActiveRecordSerialize
+        sha: Gitlab::Database::ShaAttribute.serialize(sha), # rubocop:disable Cop/ActiveRecordSerialize
         authored_date: Gitlab::Database.sanitize_timestamp(commit_hash[:authored_date]),
         committed_date: Gitlab::Database.sanitize_timestamp(commit_hash[:committed_date])
       )
@@ -27,16 +26,4 @@ class MergeRequestDiffCommit < ApplicationRecord
     Gitlab::Database.bulk_insert(self.table_name, rows)
   end
-
-  def to_hash
-    Gitlab::Git::Commit::SERIALIZE_KEYS.each_with_object({}) do |key, hash|
-      hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend
-    end
-  end
-
-  # We don't save these, because they would need a table or a serialised
-  # field. They aren't used anywhere, so just pretend the commit has no parents.
-  def parent_ids
-    []
-  end
 end

app/models/project.rb

@@ -763,6 +763,10 @@ class Project < ApplicationRecord
     Feature.enabled?(:unlink_fork_network_upon_visibility_decrease, self, default_enabled: true)
   end
+  def context_commits_enabled?
+    Feature.enabled?(:context_commits, default_enabled: true)
+  end
+
   def empty_repo?
     repository.empty?
   end

app/serializers/diffs_entity.rb

@@ -24,6 +24,10 @@ class DiffsEntity < Grape::Entity
     )
   end
+  expose :context_commits, using: API::Entities::Commit, if: -> (diffs, options) { merge_request&.project&.context_commits_enabled? } do |diffs|
+    options[:context_commits]
+  end
+
   expose :merge_request_diff, using: MergeRequestDiffEntity do |diffs|
     options[:merge_request_diff]
   end

app/services/clusters/applications/base_service.rb

@@ -58,7 +58,7 @@ module Clusters
       end
       def instantiate_application
-        raise_invalid_application_error if invalid_application?
+        raise_invalid_application_error if unknown_application?
         builder || raise(InvalidApplicationError, "invalid application: #{application_name}")
       end
@@ -67,10 +67,6 @@ module Clusters
         raise(InvalidApplicationError, "invalid application: #{application_name}")
       end
-      def invalid_application?
-        unknown_application? || (application_name == Applications::ElasticStack.application_name && !Feature.enabled?(:enable_cluster_application_elastic_stack))
-      end
-
       def unknown_application?
         Clusters::Cluster::APPLICATIONS.keys.exclude?(application_name)
       end

app/services/merge_requests/add_context_service.rb

+# frozen_string_literal: true
+
+module MergeRequests
+  class AddContextService < MergeRequests::BaseService
+    def execute
+      return error("You are not allowed to access the requested resource", 403) unless current_user&.can?(:update_merge_request, merge_request)
+      return error("Context commits: #{duplicates} are already created", 400) unless duplicates.empty?
+      return error("One or more context commits' sha is not valid.", 400) if commits.size != commit_ids.size
+
+      context_commit_ids = []
+      MergeRequestContextCommit.transaction do
+        context_commit_ids = MergeRequestContextCommit.bulk_insert(context_commit_rows, return_ids: true)
+        MergeRequestContextCommitDiffFile.bulk_insert(diff_rows(context_commit_ids))
+      end
+
+      commits
+    end
+
+    private
+
+    def raw_repository
+      project.repository.raw_repository
+    end
+
+    def merge_request
+      params[:merge_request]
+    end
+
+    def commit_ids
+      params[:commits]
+    end
+
+    def commits
+      project.repository.commits_by(oids: commit_ids)
+    end
+
+    def context_commit_rows
+      @context_commit_rows ||= build_context_commit_rows(merge_request.id, commits)
+    end
+
+    def diff_rows(context_commit_ids)
+      @diff_rows ||= build_diff_rows(raw_repository, commits, context_commit_ids)
+    end
+
+    def encode_in_base64?(diff_text)
+      (diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?) ||
+        diff_text.include?("\0")
+    end
+
+    def duplicates
+      existing_oids = merge_request.merge_request_context_commits.map { |commit| commit.sha.to_s }
+      duplicate_oids = existing_oids.select do |existing_oid|
+        commit_ids.select { |commit_id| existing_oid.start_with?(commit_id) }.count > 0
+      end
+
+      duplicate_oids
+    end
+
+    def build_context_commit_rows(merge_request_id, commits)
+      commits.map.with_index do |commit, index|
+        # generate context commit information for given commit
+        commit_hash = commit.to_hash.except(:parent_ids)
+        sha = Gitlab::Database::ShaAttribute.serialize(commit_hash.delete(:id))
+        commit_hash.merge(
+          merge_request_id: merge_request_id,
+          relative_order: index,
+          sha: sha,
+          authored_date: Gitlab::Database.sanitize_timestamp(commit_hash[:authored_date]),
+          committed_date: Gitlab::Database.sanitize_timestamp(commit_hash[:committed_date])
+        )
+      end
+    end
+
+    def build_diff_rows(raw_repository, commits, context_commit_ids)
+      diff_rows = []
+      diff_order = 0
+
+      commits.flat_map.with_index do |commit, index|
+        commit_hash = commit.to_hash.except(:parent_ids)
+        sha = Gitlab::Database::ShaAttribute.serialize(commit_hash.delete(:id))
+        # generate context commit diff information for given commit
+        diffs = commit.diffs
+
+        compare = Gitlab::Git::Compare.new(
+          raw_repository,
+          diffs.diff_refs.start_sha,
+          diffs.diff_refs.head_sha
+        )
+        compare.diffs.map do |diff|
+          diff_hash = diff.to_hash.merge(
+            sha: sha,
+            binary: false,
+            merge_request_context_commit_id: context_commit_ids[index],
+            relative_order: diff_order
+          )
+
+          # Compatibility with old diffs created with Psych.
+          diff_hash.tap do |hash|
+            diff_text = hash[:diff]
+
+            if encode_in_base64?(diff_text)
+              hash[:binary] = true
+              hash[:diff] = [diff_text].pack('m0')
+            end
+          end
+
+          # Increase order for commit so when present the diffs we can use it to keep order
+          diff_order += 1
+          diff_rows << diff_hash
+        end
+      end
+
+      diff_rows
+    end
+  end
+end

app/views/layouts/nav/sidebar/_project.html.haml

@@ -209,7 +209,7 @@
                     %span
                       = _('Artifacts')
-            - if !should_display_analytics_pages_in_sidebar && project_nav_tab?(:pipelines)
+            - if project_nav_tab?(:pipelines)
               = nav_link(controller: :pipeline_schedules) do
                 = link_to pipeline_schedules_path(@project), title: _('Schedules'), class: 'shortcuts-builds' do
                   %span

app/views/projects/ci/builds/_build.html.haml

@@ -46,7 +46,7 @@
       - if job.try(:trigger_request)
         %span.badge.badge-info= _('triggered')
       - if job.try(:allow_failure)
-        %span.badge.badge-danger= _('allowed to fail')
+        %span.badge.badge-warning= _('allowed to fail')
       - if job.schedulable?
         %span.badge.badge-info= s_('DelayedJobs|delayed')
       - elsif job.action?

changelogs/unreleased/ak-remove-es-ff.yml

+---
+title: Add new Elastic Stack cluster application for pod log aggregation
+merge_request: 23058
+author:
+type: added

changelogs/unreleased/allowed-to-fail-warning-color.yml

+---
+title: Changed color of allowed to fail badge from danger to warning
+merge_request: !23437
+author:
+type: changed

changelogs/unreleased/gitaly-version-v1.85.0.yml

+---
+title: Upgrade to Gitaly v1.85.0
+merge_request: 23945
+author:
+type: changed

changelogs/unreleased/sh-optimize-archive-controller-ref-names.yml

+---
+title: Optimize ref name lookups in archive downloads
+merge_request: 23890
+author:
+type: performance

config/initializers/attr_encrypted_thread_safe.rb

+# frozen_string_literal: true
+
+# As of v3.1.0, attr_encrypted is not thread-safe because all instances share the same `encrypted_attributes`
+# This was fixed in https://github.com/attr-encrypted/attr_encrypted/commit/d4ca0e2073ca6ba5035997ce25f7fc0b4bfbe39e
+# but no release was made after that so we have to patch it ourselves here
+
+module AttrEncrypted
+  module InstanceMethods
+    def encrypted_attributes
+      @encrypted_attributes ||= begin
+        duplicated = {}
+        self.class.encrypted_attributes.map { |key, value| duplicated[key] = value.dup }
+        duplicated
+      end
+    end
+  end
+end

config/routes/merge_requests.rb

@@ -18,6 +18,7 @@ resources :merge_requests, concerns: :awardable, except: [:new, :create, :show],
     scope constraints: ->(req) { req.format == :json }, as: :json do
       get :commits
       get :pipelines
+      get :context_commits
       get :diffs, to: 'merge_requests/diffs#show'
       get :diffs_batch, to: 'merge_requests/diffs#diffs_batch'
       get :diffs_metadata, to: 'merge_requests/diffs#diffs_metadata'

db/migrate/20190920224341_create_merge_request_context_commits_and_diffs.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateMergeRequestContextCommitsAndDiffs < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    create_table :merge_request_context_commits do |t|
+      t.references :merge_request, foreign_key: { on_delete: :cascade }
+      t.datetime_with_timezone :authored_date
+      t.datetime_with_timezone :committed_date
+      t.binary :sha, null: false
+      t.integer :relative_order, null: false
+      t.text :author_name
+      t.text :author_email
+      t.text :committer_name
+      t.text :committer_email
+      t.text :message
+      t.index [:merge_request_id, :sha], unique: true, name: 'index_mr_context_commits_on_merge_request_id_and_sha'
+    end
+
+    create_table :merge_request_context_commit_diff_files, id: false do |t|
+      t.references :merge_request_context_commit, foreign_key: { on_delete: :cascade }, index: { name: "idx_mr_cc_diff_files_on_mr_cc_id" }
+      t.binary :sha, null: false
+      t.integer :relative_order, null: false
+      t.string :a_mode, null: false, limit: 255
+      t.string :b_mode, null: false, limit: 255
+      t.boolean :new_file, null: false
+      t.boolean :renamed_file, null: false
+      t.boolean :deleted_file, null: false
+      t.boolean :too_large, null: false
+      t.boolean :binary
+      t.text :new_path, null: false
+      t.text :old_path, null: false
+      t.text :diff
+      t.index [:merge_request_context_commit_id, :sha], name: 'idx_mr_cc_diff_files_on_mr_cc_id_and_sha'
+    end
+  end
+end

db/schema.rb

@@ -2433,6 +2433,37 @@ ActiveRecord::Schema.define(version: 2020_01_27_090233) do
     t.index ["blocking_merge_request_id", "blocked_merge_request_id"], name: "index_mr_blocks_on_blocking_and_blocked_mr_ids", unique: true
   end
+  create_table "merge_request_context_commit_diff_files", id: false, force: :cascade do |t|
+    t.binary "sha", null: false
+    t.integer "relative_order", null: false
+    t.boolean "new_file", null: false
+    t.boolean "renamed_file", null: false
+    t.boolean "deleted_file", null: false
+    t.boolean "too_large", null: false
+    t.string "a_mode", limit: 255, null: false
+    t.string "b_mode", limit: 255, null: false
+    t.text "new_path", null: false
+    t.text "old_path", null: false
+    t.text "diff"
+    t.boolean "binary"
+    t.bigint "merge_request_context_commit_id"
+    t.index ["merge_request_context_commit_id", "sha"], name: "idx_mr_cc_diff_files_on_mr_cc_id_and_sha"
+  end
+
+  create_table "merge_request_context_commits", force: :cascade do |t|
+    t.datetime_with_timezone "authored_date"
+    t.datetime_with_timezone "committed_date"
+    t.integer "relative_order", null: false
+    t.binary "sha", null: false
+    t.text "author_name"
+    t.text "author_email"
+    t.text "committer_name"
+    t.text "committer_email"
+    t.text "message"
+    t.bigint "merge_request_id"
+    t.index ["merge_request_id", "sha"], name: "index_mr_context_commits_on_merge_request_id_and_sha", unique: true
+  end
+
   create_table "merge_request_diff_commits", id: false, force: :cascade do |t|
     t.datetime "authored_date"
     t.datetime "committed_date"
@@ -4702,6 +4733,8 @@ ActiveRecord::Schema.define(version: 2020_01_27_090233) do
   add_foreign_key "merge_request_assignees", "users", on_delete: :cascade
   add_foreign_key "merge_request_blocks", "merge_requests", column: "blocked_merge_request_id", on_delete: :cascade
   add_foreign_key "merge_request_blocks", "merge_requests", column: "blocking_merge_request_id", on_delete: :cascade
+  add_foreign_key "merge_request_context_commit_diff_files", "merge_request_context_commits", on_delete: :cascade
+  add_foreign_key "merge_request_context_commits", "merge_requests", on_delete: :cascade
   add_foreign_key "merge_request_diff_commits", "merge_request_diffs", on_delete: :cascade
   add_foreign_key "merge_request_diff_files", "merge_request_diffs", on_delete: :cascade
   add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade

doc/administration/gitaly/praefect.md

@@ -166,16 +166,19 @@ praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN'
 praefect['virtual_storages'] = {
   'praefect' => {
     'gitaly-1' => {
-      'address' => 'tcp://gitaly-1.internal:8075',
+      # Replace GITALY_URL_OR_IP below with the real address to connect to.
+      'address' => 'tcp://GITALY_URL_OR_IP:8075',
       'token'   => 'PRAEFECT_INTERNAL_TOKEN',
       'primary' => true
     },
     'gitaly-2' => {
-      'address' => 'tcp://gitaly-2.internal:8075',
+      # Replace GITALY_URL_OR_IP below with the real address to connect to.
+      'address' => 'tcp://GITALY_URL_OR_IP:8075',
       'token'   => 'PRAEFECT_INTERNAL_TOKEN'
     },
     'gitaly-3' => {
-      'address' => 'tcp://gitaly-3.internal:8075',
+      # Replace GITALY_URL_OR_IP below with the real address to connect to.
+      'address' => 'tcp://GITALY_URL_OR_IP:8075',
       'token'   => 'PRAEFECT_INTERNAL_TOKEN'
     }
   }
@@ -265,6 +268,8 @@ gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN'
 gitaly['listen_addr'] = "0.0.0.0:8075"
 git_data_dirs({
+  # Update this to the name of this Gitaly server which will be later
+  # exposed in the UI under "Admin area > Gitaly"
   "gitaly-1" => {
     "path" => "/var/opt/gitlab/git-data"
   }
@@ -301,13 +306,14 @@ is present, there should be two storages available to GitLab:
 ```ruby
 # /etc/gitlab/gitlab.rb on gitlab server
+# Replace PRAEFECT_URL_OR_IP below with real address Praefect can be accessed at.
 # Replace PRAEFECT_EXTERNAL_TOKEN below with real secret.
 git_data_dirs({
   "default" => {
     "path" => "/var/opt/gitlab/git-data"
   },
   "praefect" => {
-    "gitaly_address" => "tcp://praefect.internal:2305",
+    "gitaly_address" => "tcp://PRAEFECT_URL_OR_IP:2305",
     "gitaly_token" => 'PRAEFECT_EXTERNAL_TOKEN'
   }
 })
@@ -324,7 +330,7 @@ on the Praefect node.
 Save your changes and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
-Run `gitlab-rake gitlab:gitaly:check` to confirm that GitLab can reach Praefect.
+Run `sudo gitlab-rake gitlab:gitaly:check` to confirm that GitLab can reach Praefect.
 ### Testing Praefect

doc/administration/packages/dependency_proxy.md

-# GitLab Dependency Proxy administration **(PREMIUM ONLY)**
+# GitLab Dependency Proxy administration **(ULTIMATE ONLY)**
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11.

doc/api/merge_request_context_commits.md

+# Merge request context commits  API
+
+## List MR context commits
+
+Get a list of merge request context commits.
+
+```
+GET /projects/:id/merge_requests/:merge_request_iid/context_commits
+```
+
+Parameters:
+
+- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+- `merge_request_iid` (required) - The internal ID of the merge request
+
+```json
+[
+    {
+        "id": "4a24d82dbca5c11c61556f3b35ca472b7463187e",
+        "short_id": "4a24d82d",
+        "created_at": "2017-04-11T10:08:59.000Z",
+        "parent_ids": null,
+        "title": "Update README.md to include `Usage in testing and development`",
+        "message": "Update README.md to include `Usage in testing and development`",
+        "author_name": "Luke \"Jared\" Bennett",
+        "author_email": "lbennett@gitlab.com",
+        "authored_date": "2017-04-11T10:08:59.000Z",
+        "committer_name": "Luke \"Jared\" Bennett",
+        "committer_email": "lbennett@gitlab.com",
+        "committed_date": "2017-04-11T10:08:59.000Z",
+        "author": null,
+        "author_gravatar_url": "https://www.gravatar.com/avatar/2acf1fb99417a2b3971def5a294abbeb?s=80&d=identicon",
+        "commit_url": "http://127.0.0.1:3000/gitlab-org/gitlab-test/commit/4a24d82dbca5c11c61556f3b35ca472b7463187e",
+        "commit_path": "/gitlab-org/gitlab-test/commit/4a24d82dbca5c11c61556f3b35ca472b7463187e",
+        "description_html": "",
+        "title_html": "Update README.md to include `Usage in testing and development`",
+        "signature_html": null,
+        "pipeline_status_path": null
+    }
+]
+```
+
+## Create MR context commits
+
+Create a list of merge request context commits.
+
+```
+POST /projects/:id/merge_requests/:merge_request_iid/context_commits
+```
+
+Parameters:
+
+- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+- `merge_request_iid` (required) - The internal ID of the merge request
+
+```
+POST /projects/:id/merge_requests/
+```
+
+| Attribute                  | Type    | Required | Description                                                                     |
+| ---------                  | ----    | -------- | -----------                                                                     |
+| `commits`             | string array | yes | The context commits' sha  |
+
+```json
+[
+    {
+        "id": "6d394385cf567f80a8fd85055db1ab4c5295806f",
+        "message": "Added contributing guide\n\nSigned-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>\n",
+        "parent_ids": [
+            "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863"
+        ],
+        "authored_date": "2014-02-27T10:05:10.000+02:00",
+        "author_name": "Dmitriy Zaporozhets",
+        "author_email": "dmitriy.zaporozhets@gmail.com",
+        "committed_date": "2014-02-27T10:05:10.000+02:00",
+        "committer_name": "Dmitriy Zaporozhets",
+        "committer_email": "dmitriy.zaporozhets@gmail.com"
+    }
+]
+```
+
+## Delete MR context commits
+
+Delete a list of merge request context commits.
+
+```
+DELETE /projects/:id/merge_requests/:merge_request_iid/context_commits
+```
+
+Parameters:
+
+- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+- `merge_request_iid` (required) - The internal ID of the merge request
+
+| Attribute                  | Type    | Required | Description                                                                     |
+| ---------                  | ----    | -------- | -----------                                                                     |
+| `commits`             | string array | yes | The context commits' sha  |

doc/development/shell_scripting_guide/index.md

@@ -99,7 +99,7 @@ NOTE: **Note:**
 This is a work in progress.
 It is an [ongoing effort](https://gitlab.com/gitlab-org/gitlab-foss/issues/64016) to evaluate different tools for the
-automated testing of shell scripts (like [BATS](https://github.com/sstephenson/bats)).
+automated testing of shell scripts (like [BATS](https://github.com/bats-core/bats-core)).
 ## Code Review