Add latest changes from gitlab-org/gitlab@master

app/models/concerns/has_ref.rb

-# frozen_string_literal: true
-
-##
-# We will disable `ref` and `sha` attributes in `Ci::Build` in the future
-# and remove this module in favor of Ci::PipelineDelegator.
-module HasRef
-  extend ActiveSupport::Concern
-
-  def branch?
-    !tag? && !merge_request?
-  end
-
-  def git_ref
-    if branch?
-      Gitlab::Git::BRANCH_REF_PREFIX + ref.to_s
-    elsif tag?
-      Gitlab::Git::TAG_REF_PREFIX + ref.to_s
-    end
-  end
-
-  # A slugified version of the build ref, suitable for inclusion in URLs and
-  # domain names. Rules:
-  #
-  #   * Lowercased
-  #   * Anything not matching [a-z0-9-] is replaced with a -
-  #   * Maximum length is 63 bytes
-  #   * First/Last Character is not a hyphen
-  def ref_slug
-    Gitlab::Utils.slugify(ref.to_s)
-  end
-end

app/serializers/group_variable_entity.rb

@@ -4,6 +4,7 @@ class GroupVariableEntity < Grape::Entity
   expose :id
   expose :key
   expose :value
+  expose :variable_type
   expose :protected?, as: :protected
   expose :masked?, as: :masked

app/views/admin/application_settings/_ci_cd.html.haml

@@ -58,6 +58,6 @@
       = f.text_field :default_ci_config_path, class: 'form-control', placeholder: '.gitlab-ci.yml'
       %p.form-text.text-muted
         = _("The default CI configuration path for new projects.").html_safe
-        = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'custom-ci-configuration-path'), target: '_blank'
+        = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'custom-ci-configuration-path'), target: '_blank'
   = f.submit _('Save changes'), class: "btn btn-success"

app/views/projects/settings/ci_cd/_form.html.haml

@@ -8,7 +8,7 @@
             = _("Git strategy for pipelines")
           %p
             = _("Choose between <code>clone</code> or <code>fetch</code> to get the recent application code").html_safe
-            = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'git-strategy'), target: '_blank'
+            = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'git-strategy'), target: '_blank'
           .form-check
             = f.radio_button :build_allow_git_fetch, 'false', { class: 'form-check-input' }
             = f.label :build_allow_git_fetch_false, class: 'form-check-label' do
@@ -38,7 +38,7 @@
           = f.text_field :build_timeout_human_readable, class: 'form-control'
           %p.form-text.text-muted
             = _('If any job surpasses this timeout threshold, it will be marked as failed. Human readable time input language is accepted like "1 hour". Values without specification represent seconds.')
-            = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'timeout'), target: '_blank'
+            = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'timeout'), target: '_blank'
         - if can?(current_user, :update_max_artifacts_size, @project)
           %hr
@@ -55,7 +55,7 @@
           = f.text_field :ci_config_path, class: 'form-control', placeholder: '.gitlab-ci.yml'
           %p.form-text.text-muted
             = _("The path to the CI configuration file. Defaults to <code>.gitlab-ci.yml</code>").html_safe
-            = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'custom-ci-configuration-path'), target: '_blank'
+            = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'custom-ci-configuration-path'), target: '_blank'
         %hr
         .form-group
@@ -65,7 +65,7 @@
               %strong= _("Public pipelines")
             .form-text.text-muted
               = _("Allow public access to pipelines and job details, including output logs and artifacts")
-              = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'visibility-of-pipelines'), target: '_blank'
+              = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'visibility-of-pipelines'), target: '_blank'
             .bs-callout.bs-callout-info
               %p #{_("If enabled")}:
               %ul
@@ -86,7 +86,7 @@
               %strong= _("Auto-cancel redundant, pending pipelines")
             .form-text.text-muted
               = _("New pipelines will cancel older, pending pipelines on the same branch")
-              = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'auto-cancel-pending-pipelines'), target: '_blank'
+              = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'auto-cancel-pending-pipelines'), target: '_blank'
         .form-group
           .form-check
@@ -95,7 +95,7 @@
               %strong= _("Skip older, pending deployment jobs")
             .form-text.text-muted
               = _("When a deployment job is successful, skip older deployment jobs that are still pending")
-              = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'skip-older-pending-deployment-jobs'), target: '_blank'
+              = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'skip-older-pending-deployment-jobs'), target: '_blank'
         %hr
         .form-group
@@ -108,7 +108,7 @@
               .input-group-text /
           %p.form-text.text-muted
             = _("A regular expression that will be used to find the test coverage output in the job log. Leave blank to disable")
-            = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'test-coverage-parsing'), target: '_blank'
+            = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'test-coverage-parsing'), target: '_blank'
           .bs-callout.bs-callout-info
             %p= _("Below are examples of regex for existing tools:")
             %ul

changelogs/unreleased/32046-differentiate-between-errors-failures-in-xunit-result.yml

+---
+title: Differentiate between errors and failures in xUnit result
+merge_request: 23476
+author:
+type: changed

changelogs/unreleased/georgekoltsov-fix-relations-order-on-export.yml

+---
+title: Reorder exported relations by primary_key when using Project Export
+merge_request: 27117
+author:
+type: fixed

changelogs/unreleased/open-project-integration-2.yml

+---
+title: Add migration for creating open_project_tracker_data table
+merge_request: 26966
+author:
+type: other

changelogs/unreleased/repository-contributors-group-by-email.yml

+---
+title: Group repository contributors by email instead of name
+merge_request: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26899
+author: Hilco van der Wilk
+type: changed

db/migrate/20200309162244_add_open_project_tracker_data.rb

+# frozen_string_literal: true
+
+# See https://docs.gitlab.com/ee/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddOpenProjectTrackerData < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    create_table :open_project_tracker_data do |t|
+      t.references :service, foreign_key: { on_delete: :cascade }, type: :integer, index: true, null: false
+      t.timestamps_with_timezone
+      t.string :encrypted_url, limit: 255
+      t.string :encrypted_url_iv, limit: 255
+      t.string :encrypted_api_url, limit: 255
+      t.string :encrypted_api_url_iv, limit: 255
+      t.string :encrypted_token, limit: 255
+      t.string :encrypted_token_iv, limit: 255
+      t.string :closed_status_id, limit: 5
+      t.string :project_identifier_code, limit: 100
+    end
+  end
+end

db/post_migrate/20191115115043_migrate_epic_mentions_to_db.rb

 # frozen_string_literal: true
 class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
-  include Gitlab::Database::MigrationHelpers
-
   DOWNTIME = false
   disable_ddl_transaction!
@@ -28,7 +26,7 @@ class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
       .where(QUERY_CONDITIONS)
       .each_batch(of: BATCH_SIZE) do |batch, index|
       range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
-      migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
+      BackgroundMigrationWorker.perform_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
     end
   end

db/post_migrate/20200214173000_cleanup_empty_epic_user_mentions.rb

-# frozen_string_literal: true
-
-class CleanupEmptyEpicUserMentions < ActiveRecord::Migration[5.2]
-  DOWNTIME = false
-  BATCH_SIZE = 10000
-
-  class EpicUserMention < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'epic_user_mentions'
-  end
-
-  def up
-    return unless Gitlab.ee?
-
-    # cleanup epic user mentions with no actual mentions,
-    # re https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24586#note_285982468
-    EpicUserMention
-      .where(mentioned_users_ids: nil)
-      .where(mentioned_groups_ids: nil)
-      .where(mentioned_projects_ids: nil)
-      .each_batch(of: BATCH_SIZE) do |batch|
-      batch.delete_all
-    end
-  end
-
-  def down
-    # no-op
-  end
-end

db/post_migrate/20200214174519_remigrate_epic_mentions_to_db.rb

-# frozen_string_literal: true
-
-class RemigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-
-  disable_ddl_transaction!
-
-  DELAY = 2.minutes.to_i
-  BATCH_SIZE = 10000
-  MIGRATION = 'UserMentions::CreateResourceUserMention'
-
-  JOIN = "LEFT JOIN epic_user_mentions on epics.id = epic_user_mentions.epic_id"
-  QUERY_CONDITIONS = "(description like '%@%' OR title like '%@%') AND epic_user_mentions.epic_id is null"
-
-  class Epic < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'epics'
-  end
-
-  def up
-    return unless Gitlab.ee?
-
-    Epic
-      .joins(JOIN)
-      .where(QUERY_CONDITIONS)
-      .each_batch(of: BATCH_SIZE) do |batch, index|
-      range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
-      migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
-    end
-  end
-
-  def down
-    # no-op
-  end
-end

db/post_migrate/20200214174607_remigrate_epic_notes_mentions_to_db.rb

-# frozen_string_literal: true
-
-class RemigrateEpicNotesMentionsToDb < ActiveRecord::Migration[5.2]
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-
-  disable_ddl_transaction!
-
-  DELAY = 2.minutes.to_i
-  BATCH_SIZE = 10000
-  MIGRATION = 'UserMentions::CreateResourceUserMention'
-
-  INDEX_NAME = 'epic_mentions_temp_index'
-  INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Epic'"
-  QUERY_CONDITIONS = "#{INDEX_CONDITION} AND epic_user_mentions.epic_id IS NULL"
-  JOIN = 'INNER JOIN epics ON epics.id = notes.noteable_id LEFT JOIN epic_user_mentions ON notes.id = epic_user_mentions.note_id'
-
-  class Note < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'notes'
-  end
-
-  def up
-    return unless Gitlab.ee?
-
-    # create temporary index for notes with mentions, may take well over 1h
-    add_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
-
-    Note
-      .joins(JOIN)
-      .where(QUERY_CONDITIONS)
-      .each_batch(of: BATCH_SIZE) do |batch, index|
-      range = batch.pluck(Arel.sql('MIN(notes.id)'), Arel.sql('MAX(notes.id)')).first
-      migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, true, *range])
-    end
-  end
-
-  def down
-    # no-op
-    # temporary index is to be dropped in a different migration in an upcoming release:
-    # https://gitlab.com/gitlab-org/gitlab/issues/196842
-  end
-end

db/schema.rb

@@ -2921,6 +2921,21 @@ ActiveRecord::Schema.define(version: 2020_03_11_165635) do
     t.index ["access_grant_id"], name: "index_oauth_openid_requests_on_access_grant_id"
   end
+  create_table "open_project_tracker_data", force: :cascade do |t|
+    t.integer "service_id", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.string "encrypted_url", limit: 255
+    t.string "encrypted_url_iv", limit: 255
+    t.string "encrypted_api_url", limit: 255
+    t.string "encrypted_api_url_iv", limit: 255
+    t.string "encrypted_token", limit: 255
+    t.string "encrypted_token_iv", limit: 255
+    t.string "closed_status_id", limit: 5
+    t.string "project_identifier_code", limit: 100
+    t.index ["service_id"], name: "index_open_project_tracker_data_on_service_id"
+  end
+
   create_table "operations_feature_flag_scopes", force: :cascade do |t|
     t.bigint "feature_flag_id", null: false
     t.datetime_with_timezone "created_at", null: false
@@ -4993,6 +5008,7 @@ ActiveRecord::Schema.define(version: 2020_03_11_165635) do
   add_foreign_key "notes", "reviews", name: "fk_2e82291620", on_delete: :nullify
   add_foreign_key "notification_settings", "users", name: "fk_0c95e91db7", on_delete: :cascade
   add_foreign_key "oauth_openid_requests", "oauth_access_grants", column: "access_grant_id", name: "fk_77114b3b09", on_delete: :cascade
+  add_foreign_key "open_project_tracker_data", "services", on_delete: :cascade
   add_foreign_key "operations_feature_flag_scopes", "operations_feature_flags", column: "feature_flag_id", on_delete: :cascade
   add_foreign_key "operations_feature_flags", "projects", on_delete: :cascade
   add_foreign_key "operations_feature_flags_clients", "projects", on_delete: :cascade

doc/README.md

@@ -238,7 +238,7 @@ The following documentation relates to the DevOps **Verify** stage:
 | [GitLab CI/CD](ci/README.md)                                                      | Explore the features and capabilities of Continuous Integration with GitLab.                            |
 | [JUnit test reports](ci/junit_test_reports.md)                                    | Display JUnit test reports on merge requests.                                                           |
 | [Multi-project pipelines](ci/multi_project_pipelines.md) **(PREMIUM)**            | Visualize entire pipelines that span multiple projects, including all cross-project inter-dependencies. |
-| [Pipeline Graphs](ci/pipelines.md#visualizing-pipelines)                          | Visualize builds.                                                                                       |
+| [Pipeline Graphs](ci/pipelines/index.md#visualizing-pipelines)                          | Visualize builds.                                                                                       |
 | [Review Apps](ci/review_apps/index.md)                                            | Preview changes to your application right from a merge request.                                         |
 <div align="right">

doc/administration/compliance.md

@@ -17,4 +17,4 @@ GitLab’s [security features](../security/README.md) may also help you meet rel
 |**[Audit logs](audit_events.md)**<br>To maintain the integrity of your code, GitLab Enterprise Edition Premium gives admins the ability to view any modifications made within the GitLab server in an advanced audit log system, so you can control, analyze, and track every change.|Premium+||
 |**[Auditor users](auditor_users.md)**<br>Auditor users are users who are given read-only access to all projects, groups, and other resources on the GitLab instance.|Premium+||
 |**[Credentials inventory](../user/admin_area/credentials_inventory.md)**<br>With a credentials inventory, GitLab administrators can keep track of the credentials used by all of the users in their GitLab instance. |Ultimate||
-|**Separation of Duties using [Protected branches](../user/project/protected_branches.md#protected-branches-approval-by-code-owners-premium) and [custom CI Configuration Paths](../user/project/pipelines/settings.md#custom-ci-configuration-path)**<br> GitLab Silver and Premium users can leverage GitLab's cross-project YAML configuration's to define deployers of code and developers of code. View the [Separation of Duties Deploy Project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md) and [Separation of Duties Project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md) to see how to use this set up to define these roles.|Premium+||
+|**Separation of Duties using [Protected branches](../user/project/protected_branches.md#protected-branches-approval-by-code-owners-premium) and [custom CI Configuration Paths](../ci/pipelines/settings.md#custom-ci-configuration-path)**<br> GitLab Silver and Premium users can leverage GitLab's cross-project YAML configuration's to define deployers of code and developers of code. View the [Separation of Duties Deploy Project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md) and [Separation of Duties Project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md) to see how to use this set up to define these roles.|Premium+||

doc/api/projects.md

@@ -1174,7 +1174,7 @@ PUT /projects/:id
 | `auto_cancel_pending_pipelines` | string | no | Auto-cancel pending pipelines (Note: this is not a boolean, but enabled/disabled |
 | `build_coverage_regex` | string | no | Test coverage parsing |
 | `ci_config_path` | string | no | The path to CI config file |
-| `ci_default_git_depth` | integer | no | Default number of revisions for [shallow cloning](../user/project/pipelines/settings.md#git-shallow-clone) |
+| `ci_default_git_depth` | integer | no | Default number of revisions for [shallow cloning](../ci/pipelines/settings.md#git-shallow-clone) |
 | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for this project |
 | `auto_devops_deploy_strategy` | string | no | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`) |
 | `repository_storage` | string | no | Which storage shard the repository is on. Available only to admins |

doc/ci/README.md

@@ -80,13 +80,13 @@ GitLab CI/CD supports numerous configuration options:
 | Configuration | Description  |
 |:--------------|:-------------|
-| [Pipelines](pipelines.md) | Structure your CI/CD process through pipelines. |
+| [Pipelines](pipelines/index.md) | Structure your CI/CD process through pipelines. |
 | [Environment variables](variables/README.md) | Reuse values based on a variable/value key pair. |
 | [Environments](environments.md) | Deploy your application to different environments (e.g., staging, production). |
 | [Job artifacts](pipelines/job_artifacts.md) | Output, use, and reuse job artifacts. |
 | [Cache dependencies](caching/index.md) | Cache your dependencies for a faster execution. |
 | [Schedule pipelines](pipelines/schedules.md) | Schedule pipelines to run as often as you need. |
-| [Custom path for `.gitlab-ci.yml`](../user/project/pipelines/settings.md#custom-ci-configuration-path) | Define a custom path for the CI/CD configuration file. |
+| [Custom path for `.gitlab-ci.yml`](pipelines/settings.md#custom-ci-configuration-path) | Define a custom path for the CI/CD configuration file. |
 | [Git submodules for CI/CD](git_submodules.md) | Configure jobs for using Git submodules.|
 | [SSH keys for CI/CD](ssh_keys/README.md) | Using SSH keys in your CI pipelines. |
 | [Pipelines triggers](triggers/README.md) | Trigger pipelines through the API. |
@@ -149,7 +149,7 @@ As a GitLab administrator, you can change the default behavior
 of GitLab CI/CD for:
 - An [entire GitLab instance](../user/admin_area/settings/continuous_integration.md).
-- Specific projects, using [pipelines settings](../user/project/pipelines/settings.md).
+- Specific projects, using [pipelines settings](pipelines/settings.md).
 See also:

doc/ci/environments.md

@@ -47,7 +47,7 @@ can even access a [web terminal](#web-terminals) for your environment from withi
 Configuring environments involves:
-1. Understanding how [pipelines](pipelines.md) work.
+1. Understanding how [pipelines](pipelines/index.md) work.
 1. Defining environments in your project's [`.gitlab-ci.yml`](yaml/README.md) file.
 1. Creating a job configured to deploy your application. For example, a deploy job configured with [`environment`](yaml/README.md#environment) to deploy your application to a [Kubernetes cluster](../user/project/clusters/index.md).

doc/ci/examples/test-scala-application.md

@@ -60,7 +60,7 @@ You can use other versions of Scala and SBT by defining them in
 Add the `Coverage was \[\d+.\d+\%\]` regular expression in the
 **Settings ➔ Pipelines ➔ Coverage report** project setting to
-retrieve the [test coverage](../../user/project/pipelines/settings.md#test-coverage-report-badge)
+retrieve the [test coverage](../pipelines/settings.md#test-coverage-report-badge)
 rate from the build trace and have it displayed with your jobs.
 **Pipelines** must be enabled for this option to appear.