Add latest changes from gitlab-org/gitlab@master

changelogs/unreleased/chore-admin-mode-rack-attack-default-paths-migration.yml

+---
+title: Add admin mode controller path to Rack::Attack defaults
+merge_request: 20735
+author: Diego Louzán
+type: changed

changelogs/unreleased/feat-circuit-project-service.yml

+---
+title: Add Unify Circuit project integration service
+merge_request: 19849
+author: Fabio Huser
+type: added

changelogs/unreleased/filter-for-project-and-group-audit-events.yml

+---
+title: Add created_before/after filter to group/project audit events
+merge_request: 20641
+author:
+type: added

changelogs/unreleased/fj-31133-snippet-content-size-limit.yml

+---
+title: Add limit for snippet content size
+merge_request: 20346
+author:
+type: performance

changelogs/unreleased/helm_values_default.yml

+---
+title: Upgrade auto-deploy-image for helm default values file
+merge_request: 20588
+author:
+type: changed

db/migrate/20191118173522_add_snippet_size_limit_to_application_settings.rb

+# frozen_string_literal: true
+
+class AddSnippetSizeLimitToApplicationSettings < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def up
+    add_column :application_settings, :snippet_size_limit, :bigint, default: 50.megabytes, null: false
+  end
+
+  def down
+    remove_column :application_settings, :snippet_size_limit
+  end
+end

db/migrate/20191125114345_add_admin_mode_protected_path.rb

+# frozen_string_literal: true
+
+class AddAdminModeProtectedPath < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  ADMIN_MODE_ENDPOINT = '/admin/session'
+
+  OLD_DEFAULT_PROTECTED_PATHS = [
+      '/users/password',
+      '/users/sign_in',
+      '/api/v3/session.json',
+      '/api/v3/session',
+      '/api/v4/session.json',
+      '/api/v4/session',
+      '/users',
+      '/users/confirmation',
+      '/unsubscribes/',
+      '/import/github/personal_access_token'
+  ]
+
+  NEW_DEFAULT_PROTECTED_PATHS = OLD_DEFAULT_PROTECTED_PATHS.dup << ADMIN_MODE_ENDPOINT
+
+  class ApplicationSetting < ActiveRecord::Base
+    self.table_name = 'application_settings'
+  end
+
+  def up
+    change_column_default :application_settings, :protected_paths, NEW_DEFAULT_PROTECTED_PATHS
+
+    # schema allows nulls for protected_paths
+    ApplicationSetting.where.not(protected_paths: nil).each do |application_setting|
+      unless application_setting.protected_paths.include?(ADMIN_MODE_ENDPOINT)
+        updated_protected_paths = application_setting.protected_paths << ADMIN_MODE_ENDPOINT
+
+        application_setting.update(protected_paths: updated_protected_paths)
+      end
+    end
+  end
+
+  def down
+    change_column_default :application_settings, :protected_paths, OLD_DEFAULT_PROTECTED_PATHS
+
+    # schema allows nulls for protected_paths
+    ApplicationSetting.where.not(protected_paths: nil).each do |application_setting|
+      if application_setting.protected_paths.include?(ADMIN_MODE_ENDPOINT)
+        updated_protected_paths = application_setting.protected_paths - [ADMIN_MODE_ENDPOINT]
+
+        application_setting.update(protected_paths: updated_protected_paths)
+      end
+    end
+  end
+end

db/schema.rb

@@ -328,7 +328,7 @@ ActiveRecord::Schema.define(version: 2019_11_25_140458) do
     t.boolean "throttle_protected_paths_enabled", default: false, null: false
     t.integer "throttle_protected_paths_requests_per_period", default: 10, null: false
     t.integer "throttle_protected_paths_period_in_seconds", default: 60, null: false
-    t.string "protected_paths", limit: 255, default: ["/users/password", "/users/sign_in", "/api/v3/session.json", "/api/v3/session", "/api/v4/session.json", "/api/v4/session", "/users", "/users/confirmation", "/unsubscribes/", "/import/github/personal_access_token"], array: true
+    t.string "protected_paths", limit: 255, default: ["/users/password", "/users/sign_in", "/api/v3/session.json", "/api/v3/session", "/api/v4/session.json", "/api/v4/session", "/users", "/users/confirmation", "/unsubscribes/", "/import/github/personal_access_token", "/admin/session"], array: true
     t.boolean "throttle_incident_management_notification_enabled", default: false, null: false
     t.integer "throttle_incident_management_notification_period_in_seconds", default: 3600
     t.integer "throttle_incident_management_notification_per_period", default: 3600
@@ -361,6 +361,7 @@ ActiveRecord::Schema.define(version: 2019_11_25_140458) do
     t.string "encrypted_slack_app_secret_iv", limit: 255
     t.text "encrypted_slack_app_verification_token"
     t.string "encrypted_slack_app_verification_token_iv", limit: 255
+    t.bigint "snippet_size_limit", default: 52428800, null: false
     t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id"
     t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id"
     t.index ["instance_administration_project_id"], name: "index_applicationsettings_on_instance_administration_project_id"

doc/administration/index.md

@@ -158,6 +158,10 @@ Learn how to install, configure, update, and maintain your GitLab instance.
 - [Shared Runners pipelines quota](../user/admin_area/settings/continuous_integration.md#shared-runners-pipeline-minutes-quota-starter-only): Limit the usage of pipeline minutes for Shared Runners. **(STARTER ONLY)**
 - [Enable/disable Auto DevOps](../topics/autodevops/index.md#enablingdisabling-auto-devops): Enable or disable Auto DevOps for your instance.
+## Snippet settings
+
+- [Setting snippet content size limit](snippets/index.md): Set a maximum size limit for snippets' content.
+
 ## Git configuration options
 - [Custom Git hooks](custom_hooks.md): Custom Git hooks (on the filesystem) for when webhooks aren't enough.

doc/administration/snippets/index.md

+---
+type: reference, howto
+---
+
+# Snippets settings **(CORE ONLY)**
+
+Adjust the snippets' settings of your GitLab instance.
+
+## Snippets content size limit
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31133) in GitLab 12.6.
+
+You can set a content size max limit in snippets. This limit can prevent
+abuses of the feature. The default content size limit is **52428800 Bytes** (50MB).
+
+### How does it work?
+
+The content size limit will be applied when a snippet is created or
+updated. Nevertheless, in order not to break any existing snippet,
+the limit will only be enforced in stored snippets when the content
+is updated.
+
+### Snippets size limit configuration
+
+This setting is not available through the [Admin Area settings](../../user/admin_area/settings/index.md).
+In order to configure this setting, use either the Rails console
+or the [Application settings API](../../api/settings.md).
+
+NOTE: **IMPORTANT:**
+The value of the limit **must** be in Bytes.
+
+#### Through the Rails console
+
+The steps to configure this setting through the Rails console are:
+
+1. Start the Rails console:
+
+   ```bash
+   # For Omnibus installations
+   sudo gitlab-rails console
+
+   # For installations from source
+   sudo -u git -H bundle exec rails console production
+   ```
+
+1. Update the snippets maximum file size:
+
+   ```ruby
+   ApplicationSetting.first.update!(snippet_size_limit: 50.megabytes)
+   ```
+
+To retrieve the current value, start the Rails console and run:
+
+  ```ruby
+  Gitlab::CurrentSettings.snippet_size_limit
+  ```
+
+#### Through the API
+
+The process to set the snippets size limit through the Application Settings API is
+exactly the same as you would do to [update any other setting](../../api/settings.md#change-application-settings).
+
+```bash
+curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/settings?snippet_size_limit=52428800
+```
+
+You can also use the API to [retrieve the current value](../../api/settings.md#get-current-application-settings).
+
+```bash
+curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/settings
+```

doc/api/services.md

@@ -229,6 +229,51 @@ Get Campfire service settings for a project.
 GET /projects/:id/services/campfire
 ```
+## Unify Circuit
+
+Unify Circuit RTC and collaboration tool.
+
+### Create/Edit Unify Circuit service
+
+Set Unify Circuit service for a project.
+
+```
+PUT /projects/:id/services/unify-circuit
+```
+
+Parameters:
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `webhook` | string | true | The Unify Circuit webhook. For example, `https://circuit.com/rest/v2/webhooks/incoming/...`. |
+| `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines |
+| `branches_to_be_notified` | string | all | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected" |
+| `push_events` | boolean | false | Enable notifications for push events |
+| `issues_events` | boolean | false | Enable notifications for issue events |
+| `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events |
+| `merge_requests_events` | boolean | false | Enable notifications for merge request events |
+| `tag_push_events` | boolean | false | Enable notifications for tag push events |
+| `note_events` | boolean | false | Enable notifications for note events |
+| `confidential_note_events` | boolean | false | Enable notifications for confidential note events |
+| `pipeline_events` | boolean | false | Enable notifications for pipeline events |
+| `wiki_page_events` | boolean | false | Enable notifications for wiki page events |
+
+### Delete Unify Circuit service
+
+Delete Unify Circuit service for a project.
+
+```
+DELETE /projects/:id/services/unify-circuit
+```
+
+### Get Unify Circuit service settings
+
+Get Unify Circuit service settings for a project.
+
+```
+GET /projects/:id/services/unify-circuit
+```
+
 ## Custom Issue Tracker
 Custom issue tracker
@@ -480,6 +525,7 @@ Parameters:
 | `merge_requests_events` | boolean | false | Enable notifications for merge request events |
 | `tag_push_events` | boolean | false | Enable notifications for tag push events |
 | `note_events` | boolean | false | Enable notifications for note events |
+| `confidential_note_events` | boolean | false | Enable notifications for confidential note events |
 | `pipeline_events` | boolean | false | Enable notifications for pipeline events |
 | `wiki_page_events` | boolean | false | Enable notifications for wiki page events |
@@ -1088,6 +1134,7 @@ Parameters:
 | `merge_requests_events` | boolean | false | Enable notifications for merge request events |
 | `tag_push_events` | boolean | false | Enable notifications for tag push events |
 | `note_events` | boolean | false | Enable notifications for note events |
+| `confidential_note_events` | boolean | false | Enable notifications for confidential note events |
 | `pipeline_events` | boolean | false | Enable notifications for pipeline events |
 | `wiki_page_events` | boolean | false | Enable notifications for wiki page events |
 | `push_channel` | string | false | The name of the channel to receive push events notifications |
@@ -1095,6 +1142,7 @@ Parameters:
 | `confidential_issue_channel` | string | false | The name of the channel to receive confidential issues events notifications |
 | `merge_request_channel` | string | false | The name of the channel to receive merge request events notifications |
 | `note_channel` | string | false | The name of the channel to receive note events notifications |
+| `confidential_note_channel` | boolean | The name of the channel to receive confidential note events notifications |
 | `tag_push_channel` | string | false | The name of the channel to receive tag push events notifications |
 | `pipeline_channel` | string | false | The name of the channel to receive pipeline events notifications |
 | `wiki_page_channel` | string | false | The name of the channel to receive wiki page events notifications |

doc/api/settings.md

@@ -350,3 +350,4 @@ are listed in the descriptions of the relevant settings.
 | `user_show_add_ssh_key_message`          | boolean          | no                                   | When set to `false` disable the "You won't be able to pull or push project code via SSH" warning shown to users with no uploaded SSH key. |
 | `version_check_enabled`                  | boolean          | no                                   | Let GitLab inform you when an update is available. |
 | `web_ide_clientside_preview_enabled`     | boolean          | no                                   | Client side evaluation (allow live previews of JavaScript projects in the Web IDE using CodeSandbox client side evaluation). |
+| `snippet_size_limit`                     | integer          | no                                   | Max snippet content size in **bytes**. Default: 52428800 Bytes (50MB).|

doc/ci/jenkins/index.md

@@ -228,5 +228,5 @@ our very powerful [`only/except` rules system](../yaml/README.md#onlyexcept-basi
 ```yaml
 my_job:
-  only: branches
+  only: [branches]
 ```

doc/ci/merge_request_pipelines/index.md

@@ -30,7 +30,7 @@ Pipelines for merge requests have the following requirements and limitations:
 ## Configuring pipelines for merge requests
-To configure pipelines for merge requests, add the `only: merge_requests` parameter to
+To configure pipelines for merge requests, add the `only: [merge_requests]` parameter to
 the jobs that you want to run only for merge requests.
 Then, when developers create or update merge requests, a pipeline runs
@@ -68,7 +68,7 @@ After the merge request is updated with new commits:
 - The pipeline fetches the latest code from the source branch and run tests against it.
 In the above example, the pipeline contains only a `test` job.
-Since the `build` and `deploy` jobs don't have the `only: merge_requests` parameter,
+Since the `build` and `deploy` jobs don't have the `only: [merge_requests]` parameter,
 they will not run in the merge request.
 Pipelines tagged with the **detached** badge indicate that they were triggered
@@ -86,7 +86,7 @@ Read the [documentation on Merge Trains](pipelines_for_merged_results/merge_trai
 ## Excluding certain jobs
-The behavior of the `only: merge_requests` parameter is such that _only_ jobs with
+The behavior of the `only: [merge_requests]` parameter is such that _only_ jobs with
 that parameter are run in the context of a merge request; no other jobs will be run.
 However, you may want to reverse this behavior, having all of your jobs to run _except_

doc/ci/yaml/README.md

@@ -780,7 +780,7 @@ it is possible to define a job to be created based on files modified
 in a merge request.
 In order to deduce the correct base SHA of the source branch, we recommend combining
-this keyword with `only: merge_requests`. This way, file differences are correctly
+this keyword with `only: [merge_requests]`. This way, file differences are correctly
 calculated from any further commits, thus all changes in the merge requests are properly
 tested in pipelines.
@@ -802,7 +802,7 @@ either files in `service-one` directory or the `Dockerfile`, GitLab creates
 and triggers the `docker build service one` job.
 Note that if [pipelines for merge requests](../merge_request_pipelines/index.md) is
-combined with `only: change`, but `only: merge_requests` is omitted, there could be
+combined with `only: [change]`, but `only: [merge_requests]` is omitted, there could be
 unwanted behavior.
 For example:

doc/development/documentation/styleguide.md

@@ -699,7 +699,7 @@ nicely on different mobile devices.
 ## Code blocks
 - Always wrap code added to a sentence in inline code blocks (`` ` ``).
-  E.g., `.gitlab-ci.yml`, `git add .`, `CODEOWNERS`, `only: master`.
+  E.g., `.gitlab-ci.yml`, `git add .`, `CODEOWNERS`, `only: [master]`.
   File names, commands, entries, and anything that refers to code should be added to code blocks.
   To make things easier for the user, always add a full code block for things that can be
   useful to copy and paste, as they can easily do it with the button on code blocks.

doc/development/merge_request_performance_guidelines.md

@@ -165,6 +165,54 @@ can quickly spiral out of control.
 There are some cases where this may be needed. If this is the case this should
 be clearly mentioned in the merge request description.
+## Batch process
+
+**Summary:** Iterating a single process to external services (e.g. PostgreSQL, Redis, Object Storage, etc)
+should be executed in a **batch-style** in order to reduce connection overheads.
+
+For fetching rows from various tables in a batch-style, please see [Eager Loading](#eager-loading) section.
+
+### Example: Delete multiple files from Object Storage
+
+When you delete multiple files from object storage (e.g. GCS),
+executing a single REST API call multiple times is a quite expensive
+process. Ideally, this should be done in a batch-style, for example, S3 provides
+[batch deletion API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html),
+so it'd be a good idea to consider such an approach.
+
+The `FastDestroyAll` module might help this situation. It's a
+small framework when you remove a bunch of database rows and its associated data
+in a batch style.
+
+## Timeout
+
+**Summary:** You should set a reasonable timeout when the system invokes HTTP calls
+to external services (e.g. Kubernetes), and it should be executed in Sidekiq, not
+in Puma/Unicorn threads.
+
+Often, GitLab needs to communicate with an external service such as Kubernetes
+clusters. In this case, it's hard to estimate when the external service finishes
+the requested process, for example, if it's a user-owned cluster that is inactive for some reason,
+GitLab might wait for the response forever ([Example](https://gitlab.com/gitlab-org/gitlab/issues/31475)).
+This could result in Puma/Unicorn timeout and should be avoided at all cost.
+
+You should set a reasonable timeout, gracefully handle exceptions and surface the
+errors in UI or logging internally.
+
+Using [`ReactiveCaching`](https://docs.gitlab.com/ee/development/utilities.html#reactivecaching) is one of the best solutions to fetch external data.
+
+## Keep database transaction minimal
+
+**Summary:** You should avoid accessing to external services (e.g. Gitaly) during database
+transactions, otherwise it leads to severe contention problems
+as an open transaction basically blocks the release of a Postgres backend connection.
+
+For keeping transaction as minimal as possible, please consider using `AfterCommitQueue`
+module or `after_commit` AR hook.
+
+Here is [an example](https://gitlab.com/gitlab-org/gitlab/issues/36154#note_247228859)
+that one request to Gitaly instance during transaction triggered a P1 issue.
+
 ## Eager Loading
 **Summary:** always eager load associations when retrieving more than one row.

doc/security/rack_attack.md

@@ -53,7 +53,8 @@ default['gitlab']['gitlab-rails']['rack_attack_protected_paths'] = [
   '/users',
   '/users/confirmation',
   '/unsubscribes/',
-  '/import/github/personal_access_token'
+  '/import/github/personal_access_token',
+  '/admin/session'
 ]
 ```

doc/topics/autodevops/index.md

@@ -651,6 +651,8 @@ procfile exec` to replicate the environment where your application will run.
 #### Workers
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30628) in GitLab 12.6, `.gitlab/auto-deploy-values.yaml` will be used by default for Helm upgrades.
+
 Some web applications need to run extra deployments for "worker processes". For
 example, it is common in a Rails application to have a separate worker process
 to run background tasks like sending emails.
@@ -672,13 +674,18 @@ need to:
 - Set a CI variable `K8S_SECRET_REDIS_URL`, which the URL of this instance to
   ensure it's passed into your deployments.
-Once you have configured your worker to respond to health checks, you will
-need to configure a CI variable `HELM_UPGRADE_EXTRA_ARGS` with the value
-`--values helm-values.yaml`.
-Then you can, for example, run a Sidekiq worker for your Rails application
-by adding a file named `helm-values.yaml` to your repository with the following
-content:
+Once you have configured your worker to respond to health checks, run a Sidekiq
+worker for your Rails application. For:
+
+- GitLab 12.6 and later, either:
+  - Add a file named `.gitlab/auto-deploy-values.yaml` to your repository. It will
+    be automatically used if found.
+  - Add a file with a different name or path to the repository, and override the value of the
+    `HELM_UPGRADE_VALUES_FILE` variable with the path and name.
+- GitLab 12.5 and earlier, run the worker with the `--values` parameter that specifies
+  a file in the repository.
+In any case, the file must contain the following:
 ```yml
 workers:
@@ -983,6 +990,7 @@ applications.
 | `CANARY_PRODUCTION_REPLICAS`            | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md) in the production environment. Takes precedence over `CANARY_REPLICAS`. Defaults to 1. |
 | `CANARY_REPLICAS`                       | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. |
 | `HELM_RELEASE_NAME`                     | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. |
+| `HELM_UPGRADE_VALUES_FILE`              | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. |
 | `HELM_UPGRADE_EXTRA_ARGS`               | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes will not prevent word splitting. **Tip:** you can use this variable to [customize the Auto Deploy Helm chart](#custom-helm-chart) by applying custom override values with `--values my-values.yaml`. |
 | `INCREMENTAL_ROLLOUT_MODE`              | From GitLab 11.4, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production-premium) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. |
 | `K8S_SECRET_*`                          | From GitLab 11.7, any variable prefixed with [`K8S_SECRET_`](#application-secret-variables) will be made available by Auto DevOps as environment variables to the deployed application. |

doc/user/admin_area/settings/protected_paths.md

@@ -14,7 +14,8 @@ GitLab protects the following paths with Rack Attack by default:
 '/users',
 '/users/confirmation',
 '/unsubscribes/',
-'/import/github/personal_access_token'
+'/import/github/personal_access_token',
+'/admin/session'
 ```
 GitLab responds with HTTP status code `429` to POST requests at protected paths