Add latest changes from gitlab-org/gitlab@master

app/models/clusters/applications/ingress.rb

@@ -4,6 +4,7 @@ module Clusters
   module Applications
     class Ingress < ApplicationRecord
       VERSION = '1.29.7'
+      INGRESS_CONTAINER_NAME = 'nginx-ingress-controller'
       MODSECURITY_LOG_CONTAINER_NAME = 'modsecurity-log'
       self.table_name = 'clusters_applications_ingress'
@@ -69,7 +70,7 @@ module Clusters
       end
       def ingress_service
-        cluster.kubeclient.get_service('ingress-nginx-ingress-controller', Gitlab::Kubernetes::Helm::NAMESPACE)
+        cluster.kubeclient.get_service("ingress-#{INGRESS_CONTAINER_NAME}", Gitlab::Kubernetes::Helm::NAMESPACE)
       end
       private
@@ -123,7 +124,7 @@ module Clusters
               {
                 "name" => "modsecurity-template-volume",
                 "configMap" => {
-                  "name" => "ingress-nginx-ingress-controller",
+                  "name" => "ingress-#{INGRESS_CONTAINER_NAME}",
                   "items" => [
                     {
                       "key" => "modsecurity.conf",

app/models/service.rb

@@ -319,10 +319,6 @@ class Service < ApplicationRecord
     nil
   end
-  def self.find_by_template
-    find_by(template: true)
-  end
-
   # override if needed
   def supports_data_fields?
     false

app/models/user_bot_type_enums.rb

@@ -2,9 +2,6 @@
 module UserBotTypeEnums
   def self.bots
-    # When adding a new key, please ensure you are not conflicting
-    # with EE-only keys in app/models/user_type_enums.rb
-    # or app/models/user_bot_type_enums.rb
     {
       alert_bot: 2
     }

app/models/user_type_enums.rb

@@ -2,9 +2,6 @@
 module UserTypeEnums
   def self.types
-    # When adding a new key, please ensure you are not conflicting
-    # with EE-only keys in app/models/user_type_enums.rb
-    # or app/models/user_bot_type_enums.rb
     bots
   end

app/services/merge_requests/after_create_service.rb

+# frozen_string_literal: true
+
+module MergeRequests
+  class AfterCreateService < MergeRequests::BaseService
+    def execute(merge_request)
+      event_service.open_mr(merge_request, current_user)
+      notification_service.new_merge_request(merge_request, current_user)
+
+      # https://gitlab.com/gitlab-org/gitlab/issues/208813
+      if ::Feature.enabled?(:create_merge_request_pipelines_in_sidekiq, project)
+        create_pipeline_for(merge_request, current_user)
+        merge_request.update_head_pipeline
+      end
+
+      merge_request.diffs(include_stats: false).write_cache
+      merge_request.create_cross_references!(current_user)
+    end
+  end
+end

app/services/merge_requests/create_service.rb

@@ -13,19 +13,20 @@ module MergeRequests
       create(merge_request)
     end
-    def before_create(merge_request)
-      # current_user (defined in BaseService) is not available within run_after_commit block
-      user = current_user
-      merge_request.run_after_commit do
-        NewMergeRequestWorker.perform_async(merge_request.id, user.id)
-      end
-    end
-
     def after_create(issuable)
+      # Add new items to MergeRequests::AfterCreateService if they can
+      # be performed in Sidekiq
+      NewMergeRequestWorker.perform_async(issuable.id, current_user.id)
+
       todo_service.new_merge_request(issuable, current_user)
       issuable.cache_merge_request_closes_issues!(current_user)
-      create_pipeline_for(issuable, current_user)
-      issuable.update_head_pipeline
+
+      # https://gitlab.com/gitlab-org/gitlab/issues/208813
+      unless ::Feature.enabled?(:create_merge_request_pipelines_in_sidekiq, project)
+        create_pipeline_for(issuable, current_user)
+        issuable.update_head_pipeline
+      end
+
       Gitlab::UsageDataCounters::MergeRequestCounter.count(:create)
       link_lfs_objects(issuable)

app/workers/new_merge_request_worker.rb

@@ -12,11 +12,9 @@ class NewMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker
   def perform(merge_request_id, user_id)
     return unless objects_found?(merge_request_id, user_id)
-    EventCreateService.new.open_mr(issuable, user)
-    NotificationService.new.new_merge_request(issuable, user)
-
-    issuable.diffs(include_stats: false).write_cache
-    issuable.create_cross_references!(user)
+    MergeRequests::AfterCreateService
+      .new(issuable.target_project, user)
+      .execute(issuable)
   end
   def issuable_class

changelogs/unreleased/14707-add-nginx-throughput.yml

+---
+title: Add nginx request aggregations to WAF anomaly service
+merge_request: 25273
+author:
+type: added

changelogs/unreleased/create-remote-mirrors-docs.yml

+---
+title: Add documentation for create remote mirrors API
+merge_request: 26012
+author: Rajendra Kadam
+type: added

changelogs/unreleased/rs-remote-mirrors-docs.yml

+---
+title: Add remote mirrors API
+merge_request:
+author:
+type: added

doc/README.md

@@ -359,14 +359,14 @@ The following documentation relates to the DevOps **Secure** stage:
 | Secure Topics                                                                                         | Description                                                            |
 |:------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------|
-| [Compliance Dashboard](user/application_security/compliance_dashboard/index.md) **(ULTIMATE)**        | View the most recent Merge Request activity in a group.                |
+| [Compliance Dashboard](user/compliance/compliance_dashboard/index.md) **(ULTIMATE)**        | View the most recent Merge Request activity in a group.                |
 | [Container Scanning](user/application_security/container_scanning/index.md) **(ULTIMATE)**            | Use Clair to scan docker images for known vulnerabilities.             |
 | [Dependency List](user/application_security/dependency_list/index.md) **(ULTIMATE)**                  | View your project's dependencies and their known vulnerabilities.      |
 | [Dependency Scanning](user/application_security/dependency_scanning/index.md) **(ULTIMATE)**          | Analyze your dependencies for known vulnerabilities.                   |
 | [Dynamic Application Security Testing (DAST)](user/application_security/dast/index.md) **(ULTIMATE)** | Analyze running web applications for known vulnerabilities.            |
 | [Group Security Dashboard](user/application_security/security_dashboard/index.md#group-security-dashboard) **(ULTIMATE)**      | View vulnerabilities in all the projects in a group and its subgroups. |
 | [Instance Security Dashboard](user/application_security/security_dashboard/index.md#instance-security-dashboard) **(ULTIMATE)**      | View vulnerabilities in all the projects you're interested in. |
-| [License Compliance](user/application_security/license_compliance/index.md) **(ULTIMATE)**            | Search your project's dependencies for their licenses.                 |
+| [License Compliance](user/compliance/license_compliance/index.md) **(ULTIMATE)**            | Search your project's dependencies for their licenses.                 |
 | [Pipeline Security Dashboard](user/application_security/security_dashboard/index.md#pipeline-security-dashboard) **(ULTIMATE)**    | View the security reports for your project's pipelines.               |
 | [Project Security Dashboard](user/application_security/security_dashboard/index.md#project-security-dashboard) **(ULTIMATE)**    | View the latest security reports for your project.                     |
 | [Static Application Security Testing (SAST)](user/application_security/sast/index.md) **(ULTIMATE)**  | Analyze source code for known vulnerabilities.                         |

doc/administration/high_availability/README.md

@@ -20,12 +20,12 @@ watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac)
 with [John Northrup](https://gitlab.com/northrup), and live questions coming
 in from some of our customers.
-## Recommended Setups based on number of users
+## Recommended setups based on number of users
 - 1 - 1000 Users: A single-node [Omnibus](https://docs.gitlab.com/omnibus/) setup with frequent backups. Refer to the [requirements page](../../install/requirements.md) for further details of the specs you will require.
 - 2000 - 50000+ Users: A scaled HA environment based on one of our [Reference Architectures](#reference-architectures) below.
-## GitLab Components and Configuration Instructions
+## GitLab components and configuration instructions
 The GitLab application depends on the following [components](../../development/architecture.md#component-diagram)
 and services. They are included in the reference architectures along with our
@@ -48,7 +48,7 @@ in which you would typically configure them.
 In some cases, components can be combined on the same nodes to reduce complexity as well.
-## Reference Architectures
+## Reference architectures
 In this section we'll detail the Reference Architectures that can support large numbers
 of users. These were built, tested and verified by our Quality and Support teams.
@@ -69,12 +69,11 @@ how much automation you use, mirroring, and repo/change size. Additionally the
 shown memory values are given directly by [GCP machine types](https://cloud.google.com/compute/docs/machine-types).
 On different cloud vendors a best effort like for like can be used.
-### 2,000 User Configuration
-- **Supported Users (approximate):** 2,000
-- **Test RPS Rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS
-- **Known Issues:** For the latest list of known performance issues head
-[here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
+### 2,000 user configuration
+- **Supported users (approximate):** 2,000
+- **Test RPS rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS
+- **Known issues:**  [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues)
 | Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
@@ -91,12 +90,11 @@ On different cloud vendors a best effort like for like can be used.
 | External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2  |
 | Internal load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2  |
-### 5,000 User Configuration
-- **Supported Users (approximate):** 5,000
-- **Test RPS Rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS
-- **Known Issues:** For the latest list of known performance issues head
-[here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
+### 5,000 user configuration
+- **Supported users (approximate):** 5,000
+- **Test RPS rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS
+- **Known issues:**  [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues)
 | Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
@@ -113,12 +111,11 @@ On different cloud vendors a best effort like for like can be used.
 | External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2  |
 | Internal load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2  |
-### 10,000 User Configuration
-- **Supported Users (approximate):** 10,000
-- **Test RPS Rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS
-- **Known Issues:** For the latest list of known performance issues head
-[here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
+### 10,000 user configuration
+- **Supported users (approximate):** 10,000
+- **Test RPS rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS
+- **Known issues:**  [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues)
 | Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
@@ -138,12 +135,11 @@ On different cloud vendors a best effort like for like can be used.
 | External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2  |
 | Internal load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2  |
-### 25,000 User Configuration
-- **Supported Users (approximate):** 25,000
-- **Test RPS Rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS
-- **Known Issues:** For the latest list of known performance issues head
-[here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
+### 25,000 user configuration
+- **Supported users (approximate):** 25,000
+- **Test RPS rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS
+- **Known issues:**  [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues)
 | Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|
@@ -163,12 +159,11 @@ On different cloud vendors a best effort like for like can be used.
 | External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2  |
 | Internal load balancing node[^6] | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4  |
-### 50,000 User Configuration
-- **Supported Users (approximate):** 50,000
-- **Test RPS Rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS
-- **Known Issues:** For the latest list of known performance issues head
-[here](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues).
+### 50,000 user configuration
+- **Supported users (approximate):** 50,000
+- **Test RPS rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS
+- **Known issues:**  [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues)
 | Service                     | Nodes | Configuration[^8]     | GCP type      |
 | ----------------------------|-------|-----------------------|---------------|

doc/api/api_resources.md

@@ -61,6 +61,7 @@ The following API resources are available in the project context:
 | [Protected tags](protected_tags.md)                                 | `/projects/:id/protected_tags`                                                                                                                                                                        |
 | [Releases](releases/index.md)                                       | `/projects/:id/releases`                                                                                                                                                                              |
 | [Release links](releases/links.md)                                  | `/projects/:id/releases/.../assets/links`                                                                                                                                                             |
+| [Remote mirrors](remote_mirrors.md)                                 | `/projects/:id/remote_mirrors`                                                                                                                                                                        |
 | [Repositories](repositories.md)                                     | `/projects/:id/repository`                                                                                                                                                                            |
 | [Repository files](repository_files.md)                             | `/projects/:id/repository/files`                                                                                                                                                                      |
 | [Repository submodules](repository_submodules.md)                   | `/projects/:id/repository/submodules`                                                                                                                                                                 |

doc/api/remote_mirrors.md

+# Project remote mirrors API
+
+[Push mirrors](../user/project/repository/repository_mirroring.md#pushing-to-a-remote-repository-core)
+defined on a project's repository settings are called "remote mirrors", and the
+state of these mirrors can be queried and modified via the remote mirror API
+outlined below.
+
+## List a project's remote mirrors
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/38121) in GitLab 12.9.
+
+Returns an Array of remote mirrors and their statuses:
+
+```text
+GET /projects/:id/remote_mirrors
+```
+
+Example request:
+
+```sh
+curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors'
+```
+
+Example response:
+
+```json
+[
+  {
+    "enabled": true,
+    "id": 101486,
+    "last_error": null,
+    "last_successful_update_at": "2020-01-06T17:32:02.823Z",
+    "last_update_at": "2020-01-06T17:32:02.823Z",
+    "last_update_started_at": "2020-01-06T17:31:55.864Z",
+    "only_protected_branches": true,
+    "update_status": "finished",
+    "url": "https://*****:*****@gitlab.com/gitlab-org/security/gitlab.git"
+  }
+]
+```
+
+NOTE: **Note:**
+For security reasons, the `url` attribute will always be scrubbed of username
+and password information.
+
+## Create a remote mirror
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/24189) in GitLab 12.9.
+
+Create a remote mirror for a project. The mirror will be disabled by default. You can enable it by including the optional parameter `enabled` when creating it:
+
+```text
+POST /projects/:id/remote_mirrors
+```
+
+| Attribute                 | Type    | Required   | Description                                         |
+| :----------               | :-----  | :--------- | :------------                                       |
+| `url`                     | String  | yes        | The URL of the remote repository to be mirrored.    |
+| `enabled`                 | Boolean | no         | Determines if the mirror is enabled.                |
+| `only_protected_branches` | Boolean | no         | Determines if only protected branches are mirrored. |
+
+Example request:
+
+```sh
+curl --request POST --data "url=https://username:token@example.com/gitlab/example.git" --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors'
+```
+
+Example response:
+
+```json
+{
+    "enabled": false,
+    "id": 101486,
+    "last_error": null,
+    "last_successful_update_at": null,
+    "last_update_at": null,
+    "last_update_started_at": null,
+    "only_protected_branches": false,
+    "update_status": "none",
+    "url": "https://*****:*****@example.com/gitlab/example.git"
+}
+```
+
+## Update a remote mirror's attributes
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/38121) in GitLab 12.9.
+
+Toggle a remote mirror on or off, or change which types of branches are
+mirrored:
+
+```text
+PUT /projects/:id/remote_mirrors/:mirror_id
+```
+
+| Attribute                 | Type    | Required   | Description                                         |
+| :----------               | :-----  | :--------- | :------------                                       |
+| `mirror_id`               | Integer | yes        | The remote mirror ID.                               |
+| `enabled`                 | Boolean | no         | Determines if the mirror is enabled.                |
+| `only_protected_branches` | Boolean | no         | Determines if only protected branches are mirrored. |
+
+Example request:
+
+```sh
+curl --request PUT --data "enabled=false" --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors/101486'
+```
+
+Example response:
+
+```json
+{
+    "enabled": false,
+    "id": 101486,
+    "last_error": null,
+    "last_successful_update_at": "2020-01-06T17:32:02.823Z",
+    "last_update_at": "2020-01-06T17:32:02.823Z",
+    "last_update_started_at": "2020-01-06T17:31:55.864Z",
+    "only_protected_branches": true,
+    "update_status": "finished",
+    "url": "https://*****:*****@gitlab.com/gitlab-org/security/gitlab.git"
+}
+```

doc/ci/README.md

@@ -133,7 +133,7 @@ Its feature set is listed on the table below according to DevOps stages.
 | **Secure** ||
 | [Container Scanning](../user/application_security/container_scanning/index.md) **(ULTIMATE)** | Check your Docker containers for known vulnerabilities.|
 | [Dependency Scanning](../user/application_security/dependency_scanning/index.md) **(ULTIMATE)** | Analyze your dependencies for known vulnerabilities. |
-| [License Compliance](../user/application_security/license_compliance/index.md) **(ULTIMATE)** | Search your project dependencies for their licenses. |
+| [License Compliance](../user/compliance/license_compliance/index.md) **(ULTIMATE)** | Search your project dependencies for their licenses. |
 | [Security Test reports](../user/application_security/index.md) **(ULTIMATE)** | Check for app vulnerabilities. |
 ## Examples

doc/ci/examples/license_management.md

 ---
-redirect_to: '../../user/application_security/license_compliance/index.md'
+redirect_to: '../../user/compliance/license_compliance/index.md'
 ---
-This document was moved to [another location](../../user/application_security/license_compliance/index.md).
+This document was moved to [another location](../../user/compliance/license_compliance/index.md).

doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md

@@ -140,15 +140,15 @@ workaround you'd be able to take immediately. If it's not available or acceptabl
 please read through this section.
 Merge train is enabled by default when you enable [Pipelines for merged results](../index.md),
-however, you can forcibly disable this feature by disabling the feature flag `:merge_trains_enabled`.
-After you disabled this feature, all the existing merge trains will be aborted and
-you will no longer see the **Start/Add Merge Train** button in merge requests.
+however, you can disable this feature by setting the `:disable_merge_trains` feature flag to `enable`.
+When you disable this feature, all existing merge trains are aborted and
+the **Start/Add Merge Train** button no longer appears in merge requests.
 To check if the feature flag is enabled on your GitLab instance,
-please ask administrator to execute the following commands:
+please ask an administrator to execute the following commands **(CORE ONLY)**:
 ```shell
 > sudo gitlab-rails console                         # Login to Rails console of GitLab instance.
-> Feature.enabled?(:merge_trains_enabled)           # Check if it's enabled or not.
-> Feature.disable(:merge_trains_enabled)            # Disable the feature flag.
+> Feature.enabled?(:disable_merge_trains)           # Check if it's disabled or not.
+> Feature.enable(:disable_merge_trains)             # Disable Merge Trains.
 ```

doc/ci/yaml/README.md

@@ -2268,7 +2268,7 @@ introduced in GitLab 12.8.
 > Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above.
-The `license_management` report collects [Licenses](../../user/application_security/license_compliance/index.md)
+The `license_management` report collects [Licenses](../../user/compliance/license_compliance/index.md)
 as artifacts.
 The collected License Compliance report will be uploaded to GitLab as an artifact and will
@@ -2279,7 +2279,7 @@ dashboards. It is not available for download through the web interface.
 > Introduced in GitLab 12.8. Requires GitLab Runner 11.5 and above.
-The `license_scanning` report collects [Licenses](../../user/application_security/license_compliance/index.md)
+The `license_scanning` report collects [Licenses](../../user/compliance/license_compliance/index.md)
 as artifacts.
 The License Compliance report will be uploaded to GitLab as an artifact and will

doc/development/go_guide/index.md

@@ -109,7 +109,7 @@ become available, you will be able to share job templates like this
 Dependencies should be kept to the minimum. The introduction of a new
 dependency should be argued in the merge request, as per our [Approval
 Guidelines](../code_review.md#approval-guidelines). Both [License
-Management](../../user/application_security/license_compliance/index.md)
+Management](../../user/compliance/license_compliance/index.md)
 **(ULTIMATE)** and [Dependency
 Scanning](../../user/application_security/dependency_scanning/index.md)
 **(ULTIMATE)** should be activated on all projects to ensure new dependencies

doc/development/testing_guide/best_practices.md

@@ -52,6 +52,13 @@ bundle exec guard
 When using spring and guard together, use `SPRING=1 bundle exec guard` instead to make use of spring.
+Use [Factory Doctor](https://test-prof.evilmartians.io/#/factory_doctor.md) to find cases on un-necessary database manipulation, which can cause slow tests.
+
+```shell
+# run test for path
+FDOC=1 bin/rspec spec/[path]/[to]/[spec].rb
+```
+
 ### General guidelines
 - Use a single, top-level `describe ClassName` block.