Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/global.gitlab-ci.yml

@@ -13,6 +13,8 @@
 .default-before_script:
   before_script:
     - date
+    - export GOPATH=$CI_PROJECT_DIR/.go
+    - mkdir -p $GOPATH
     - source scripts/utils.sh
     - source scripts/prepare_build.sh
     - date
@@ -22,6 +24,7 @@
   cache:
     key: "debian-stretch-ruby-2.6.3-node-12.x"
     paths:
+      - .go/pkg/mod
       - vendor/ruby
       - .yarn-cache/
       - vendor/gitaly-ruby
@@ -52,7 +55,7 @@
       - "{babel.config,jest.config}.js"
       - "config.ru"
       - "{package.json,yarn.lock}"
-      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,public,rubocop,scripts,spec,symbol,vendor}/**/*"
+      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*"
       - "doc/README.md"  # Some RSpec test rely on this file
 .only-qa-changes:
@@ -83,7 +86,7 @@
       - "{babel.config,jest.config}.js"
       - "config.ru"
       - "{package.json,yarn.lock}"
-      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,public,rubocop,scripts,spec,symbol,vendor}/**/*"
+      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*"
       - "doc/README.md"  # Some RSpec test rely on this file
       - ".dockerignore"
       - "qa/**/*"
@@ -123,3 +126,4 @@
   only:
     variables:
       - $CI_PROJECT_NAME == "gitlab"
+      - $CI_PROJECT_NAME == "gitlab-ee"  # Support former project name for forks/mirrors

.gitlab/ci/rails.gitlab-ci.yml

@@ -95,6 +95,8 @@ setup-test-env:
       - tmp/tests
       - config/secrets.yml
       - vendor/gitaly-ruby
+  cache:
+    policy: pull-push
 rspec unit pg:
   extends: .rspec-base-pg

.gitlab/ci/setup.gitlab-ci.yml

@@ -43,5 +43,6 @@ no_ee_check:
     - scripts/no-ee-check
   only:
     variables:
-      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAME == "gitlab-foss"
-      - $CI_SERVER_HOST == "dev.gitlab.org" && $CI_PROJECT_NAME == "gitlabhq"
+      - $CI_PROJECT_NAME == "gitlab-foss"
+      - $CI_PROJECT_NAME == "gitlab-ce"  # Support former project name for forks/mirrors
+      - $CI_PROJECT_NAME == "gitlabhq"  # Support former project name for dev

CHANGELOG-EE.md

 Please view this file on the master branch, on stable branches it's out of date.
+## 12.3.1
+
+- No changes.
+
 ## 12.3.0
 ### Security (3 changes)

app/models/ci/pipeline.rb

@@ -771,6 +771,18 @@ module Ci
       end
     end
+    def all_worktree_paths
+      strong_memoize(:all_worktree_paths) do
+        project.repository.ls_files(sha)
+      end
+    end
+
+    def top_level_worktree_paths
+      strong_memoize(:top_level_worktree_paths) do
+        project.repository.tree(sha).blobs.map(&:path)
+      end
+    end
+
     def default_branch?
       ref == project.default_branch
     end

app/views/dashboard/projects/_zero_authorized_projects.html.haml

@@ -2,11 +2,11 @@
   .section-container.section-welcome{ class: "#{ 'section-admin-welcome' if current_user.admin? }" }
     .container.section-body
       .row
-        .blank-state-welcome
+        .blank-state-welcome.w-100
           %h2.blank-state-welcome-title
-            Welcome to GitLab
+            = _('Welcome to GitLab')
           %p.blank-state-text
-            Code, test, and deploy together
+            = _('Faster releases. Better code. Less pain.')
         .blank-state-row
           %div{ class: ('column-large' if has_start_trial?) }
             - if current_user.admin?

changelogs/unreleased/32457-change-welcome-to-gitlab-message.yml

+---
+title: Change welcome message and make translatable
+merge_request: 17391
+author:
+type: other

changelogs/unreleased/job_file_matching.yml

+---
+title: Add file matching rule to flexible CI rules
+merge_request: 16574
+author:
+type: added

doc/administration/gitaly/index.md

@@ -564,6 +564,109 @@ concurrency limiter, not a rate limiter. If a client makes 1000 requests
 in a row in a very short timespan, the concurrency will not exceed 1,
 and this mechanism (the concurrency limiter) will do nothing.
+## Rotating a Gitaly authentication token
+
+Rotating credentials in a production environment often either requires
+downtime, or causes outages, or both. If you are careful, though, you
+*can* rotate Gitaly credentials without a service interruption.
+
+This procedure also works if you are running GitLab on a single server.
+In that case, "Gitaly servers" and "Gitaly clients" refers to the same
+machine.
+
+### 1. Monitor current authentication behavior
+
+Use prometheus to see what the current authentication behavior of your
+GitLab installation is.
+
+```
+sum(rate(gitaly_authentications_total[5m])) by (enforced, status)
+```
+
+In a system where authentication is configured correctly, and where you
+have live traffic, you will see something like this:
+
+```
+{enforced="true",status="ok"}  4424.985419441742
+```
+
+There may also be other numbers with rate 0. We only care about the
+non-zero numbers.
+
+The only non-zero number should have `enforced="true",status="ok"`. If
+you have other non-zero numbers, something is wrong in your
+configuration.
+
+The 'status="ok"' number reflects your current request rate. In the example
+above, Gitaly is handling about 4000 requests per second.
+
+Now you have established that you can monitor the Gitaly authentication
+behavior of your GitLab installation.
+
+### 2. Reconfigure all Gitaly servers to be in "auth transitioning" mode
+
+The second step is to temporarily disable authentication on the Gitaly servers.
+
+```ruby
+# in /etc/gitlab/gitlab.rb
+gitaly['auth_transitioning'] = true
+```
+
+After you have applied this, your prometheus query should return
+something like this:
+
+```
+{enforced="false",status="would be ok"}  4424.985419441742
+```
+
+Because `enforced="false"`, it will be safe to start rolling out the new
+token.
+
+### 3. Update Gitaly token on all clients and servers
+
+```ruby
+# in /etc/gitlab/gitlab.rb
+
+gitaly['auth_token'] = 'my new secret token'
+```
+
+Remember to apply this on both your Gitaly clients *and* servers. If you
+check your prometheus query while this change is being rolled out, you
+will see non-zero values for the `enforced="false",status="denied"` counter.
+
+### 4. Use prometheus to ensure there are no authentication failures
+
+After you applied the Gitaly token change everywhere, and all services
+involved have been restarted, you should will temporarily see a mix of
+`status="would be ok"` and `status="denied"`.
+
+After the new token has been picked up by all Gitaly clients and
+servers, the **only non-zero rate** should be
+`enforced="false",status="would be ok"`.
+
+### 5. Disable "auth transitioning" Mode
+
+Now we turn off the 'auth transitioning' mode. These final steps are
+important: without them, you have **no authentication**.
+
+Update the configuration on your Gitaly servers:
+
+```ruby
+# in /etc/gitlab/gitlab.rb
+gitaly['auth_transitioning'] = false
+```
+
+### 6. Verify that authentication is enforced again
+
+Refresh your prometheus query. You should now see the same kind of
+result as you did in the beginning:
+
+```
+{enforced="true",status="ok"}  4424.985419441742
+```
+
+Note that `enforced="true"`, meaning that authentication is being enforced.
+
 ## Troubleshooting Gitaly
 ### `gitaly-debug`

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -715,13 +715,15 @@ Namespace.find_by_full_path("user/proj").namespace_statistics.update(shared_runn
 project = Project.find_by_full_path('')
 builds_with_artifacts =  project.builds.with_artifacts_archive
-# Prior to 10.6 the above line would be:
-# builds_with_artifacts = project.builds.with_artifacts
-
 # Instance-wide:
-builds_with_artifacts = Ci::Build.with_artifacts
+builds_with_artifacts = Ci::Build.with_artifacts_archive
+
+# Prior to 10.6 the above lines would be:
+# builds_with_artifacts =  project.builds.with_artifacts
+# builds_with_artifacts = Ci::Build.with_artifacts
 ### CLEAR THEM OUT
+# Note that this will also erase artifacts that developers marked to "Keep"
 builds_to_clear = builds_with_artifacts.where("finished_at < ?", 1.week.ago)
 builds_to_clear.each do |build|
   build.artifacts_expire_at = Time.now

doc/ci/multi_project_pipelines.md

@@ -116,7 +116,8 @@ staging:
 ```
 Use a `project` keyword to specify full path to a downstream project. Use
-a `branch` keyword to specify a branch name.
+a `branch` keyword to specify a branch name. Variable expansion is supported in
+the `branch` property.
 GitLab will use a commit that is currently on the HEAD of the branch when
 creating a downstream pipeline.

doc/development/performance.md

@@ -251,6 +251,36 @@ These results can also be placed into a PostgreSQL database by setting the
 `RSPEC_PROFILING_POSTGRES_URL` variable. This is used to profile the test suite
 when running in the CI environment.
+## Memory profiling
+
+One of the reasons of the increased memory footprint could be Ruby memory fragmentation.
+
+To diagnose it, you can visualize Ruby heap as described in [this post by Aaron Patterson](https://tenderlovemaking.com/2017/09/27/visualizing-your-ruby-heap.html).
+
+To start, you want to dump the heap of the process you are investigating to a JSON file.  
+
+You need to run the command inside the process you are exploring, you may do that with `rbtrace`.  
+`rbtrace` is already present in GitLab `Gemfile`, you just need to require it.  
+It could be achieved running webserver or Sidekiq with the environment variable set to `ENABLE_RBTRACE=1`.
+
+To get the heap dump:
+
+```ruby
+bundle exec rbtrace -p <PID> -e 'File.open("heap.json", "wb") { |t| ObjectSpace.dump_all(output: t) }'
+```
+
+Having the JSON, you finally could render a picture using the script [provided by Aaron](https://gist.github.com/tenderlove/f28373d56fdd03d8b514af7191611b88) or similar:
+
+```sh
+ruby heapviz.rb heap.json
+```
+  
+Fragmented Ruby heap snapshot could look like this:
+
+![Ruby heap fragmentation](img/memory_ruby_heap_fragmentation.png)
+
+Memory fragmentation could be reduced by tuning GC parameters as described in [this post by Nate Berkopec](https://www.speedshop.co/2017/12/04/malloc-doubles-ruby-memory.html), which should be considered as a tradeoff, as it may affect overall performance of memory allocation and GC cycles.
+
 ## Importance of Changes
 When working on performance improvements, it's important to always ask yourself

doc/development/rake_tasks.md

@@ -143,12 +143,6 @@ This will compile and minify all JavaScript and CSS assets and copy them along
 with all other frontend assets (images, fonts, etc) into `/public/assets` where
 they can be easily inspected.
-## Generate API documentation for project services (e.g. Slack)
-
-```
-bundle exec rake services:doc
-```
-
 ## Updating Emoji Aliases
 To update the Emoji aliases file (used for Emoji autocomplete) you must run the

doc/user/application_security/sast/index.md

@@ -45,12 +45,15 @@ The results are sorted by the priority of the vulnerability:
 ## Requirements
-To run a SAST job, you need GitLab Runner with the
+To run a SAST job, by default, you need GitLab Runner with the
 [`docker`](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) or
 [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html#running-privileged-containers-for-the-runners)
 executor running in privileged mode. If you're using the shared Runners on GitLab.com,
 this is enabled by default.
+Privileged mode is not necessary if you've [disabled Docker in Docker
+for SAST](#disabling-docker-in-docker-for-sast)
+
 CAUTION: **Caution:**
 If you use your own Runners, make sure that the Docker version you have installed
 is **not** `19.03.00`. See [troubleshooting information](#error-response-from-daemon-error-processing-tar-file-docker-tar-relocation-error) for details.
@@ -144,6 +147,21 @@ under your project's settings:
 | ---- | --- | ----- |
 | Variable | `MAVEN_CLI_OPTS` | `-Drepository.password=verysecret -Drepository.user=myuser` |
+### Disabling Docker in Docker for SAST
+
+You can avoid the need for Docker in Docker by running the individual analyzers.
+This does not require running the executor in privileged mode. For example:
+
+```yaml
+include:
+  template: SAST.gitlab-ci.yml
+
+variables:
+  SAST_DISABLE_DIND: "true"
+```
+
+This will create individual `<analyzer-name>-sast` jobs for each analyzer that runs in your CI/CD pipeline.
+
 ### Overriding the SAST template
 If you want to override the job definition (for example, change properties like
@@ -173,6 +191,7 @@ The following are Docker image-related variables.
 | `SAST_ANALYZER_IMAGE_PREFIX`  | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). |
 | `SAST_ANALYZER_IMAGE_TAG`     | Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md).                                 |
 | `SAST_DEFAULT_ANALYZERS`      | Override the names of default images. Read more about [customizing analyzers](analyzers.md).                                          |
+| `SAST_DISABLE_DIND`           | Disable Docker in Docker and run analyzers [individually](#disabling-docker-in-docker-for-sast).                                      |
 | `SAST_PULL_ANALYZER_IMAGES`   | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md).                 |
 ### Vulnerability filters

doc/user/project/integrations/generic_alerts.md

@@ -13,11 +13,11 @@ authored by the GitLab Alert Bot.
 ## Setting up generic alerts
-To set up the generic alerts integration:  
-1. Navigate to **Settings > Integrations** in a project.  
+To set up the generic alerts integration:
+1. Navigate to **Settings > Integrations** in a project.
 1. Click on **Alert endpoint**.
-1. Toggle the **Active**  alert setting. The `URL` and `Authorization Key` for the webhook configuration can be found there.  
+1. Toggle the **Active**  alert setting. The `URL` and `Authorization Key` for the webhook configuration can be found there.
 ## Customizing the payload
@@ -35,7 +35,11 @@ You can customize the payload by sending the following parameters. All fields ar
 Example request:
 ```sh
-curl --request POST --data '{"title": "Incident title"}' --header "Authorization: Bearer <autorization_key>" <url>
+curl --request POST \
+  --data '{"title": "Incident title"}' \
+  --header "Authorization: Bearer <autorization_key>" \
+  --header "Content-Type: application/json" \
+  <url>
 ```
 The `<autorization_key>` and `<url>` values can be found when [setting up generic alerts](#setting-up-generic-alerts).

lib/gitlab/ci/build/rules/rule/clause/exists.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause::Exists < Rules::Rule::Clause
+        # The maximum number of patterned glob comparisons that will be
+        # performed before the rule assumes that it has a match
+        MAX_PATTERN_COMPARISONS = 10_000
+
+        def initialize(globs)
+          globs = Array(globs)
+
+          @top_level_only = globs.all?(&method(:top_level_glob?))
+          @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?))
+        end
+
+        def satisfied_by?(pipeline, seed)
+          paths = worktree_paths(pipeline)
+
+          exact_matches?(paths) || pattern_matches?(paths)
+        end
+
+        private
+
+        def worktree_paths(pipeline)
+          if @top_level_only
+            pipeline.top_level_worktree_paths
+          else
+            pipeline.all_worktree_paths
+          end
+        end
+
+        def exact_matches?(paths)
+          @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } }
+        end
+
+        def pattern_matches?(paths)
+          comparisons = 0
+          @pattern_globs.any? do |glob|
+            paths.any? do |path|
+              comparisons += 1
+              comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path)
+            end
+          end
+        end
+
+        def pattern_match?(glob, path)
+          File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
+        end
+
+        # matches glob patterns that only match files in the top level directory
+        def top_level_glob?(glob)
+          !glob.include?('/') && !glob.include?('**')
+        end
+
+        # matches glob patterns that have no metacharacters for File#fnmatch?
+        def exact_glob?(glob)
+          !glob.include?('*') && !glob.include?('?') && !glob.include?('[') && !glob.include?('{')
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/rules/rule.rb

@@ -8,11 +8,11 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
-          CLAUSES      = %i[if changes].freeze
-          ALLOWED_KEYS = %i[if changes when start_in].freeze
+          CLAUSES      = %i[if changes exists].freeze
+          ALLOWED_KEYS = %i[if changes exists when start_in].freeze
           ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze
-          attributes :if, :changes, :when, :start_in
+          attributes :if, :changes, :exists, :when, :start_in
           validations do
             validates :config, presence: true
@@ -24,7 +24,7 @@ module Gitlab
             with_options allow_nil: true do
               validates :if, expression: true
-              validates :changes, array_of_strings: true
+              validates :changes, :exists, array_of_strings: true, length: { maximum: 50 }
               validates :when, allowed_values: { in: ALLOWED_WHEN }
             end
           end

lib/gitlab/danger/helper.rb

@@ -35,7 +35,8 @@ module Gitlab
       end
       def ee?
-        ENV['CI_PROJECT_NAME'] == 'gitlab' || File.exist?('../../CHANGELOG-EE.md')
+        # Support former project name for `dev` and support local Danger run
+        %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?('../../ee')
       end
       def gitlab_helper

lib/tasks/services.rake

-services_template = <<-ERB
-# Services
-
-<% services.each do |service| %>
-## <%= service[:title] %>
-
-
-<% unless service[:description].blank? %>
-<%= service[:description] %>
-<% end %>
-
-
-### Create/Edit <%= service[:title] %> service
-
-Set <%= service[:title] %> service for a project.
-<% unless service[:help].blank? %>
-
-> <%= service[:help].gsub("\n", ' ') %>
-
-<% end %>
-
-```
-PUT /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-Parameters:
-
-<% service[:params].each do |param| %>
-- `<%= param[:name] %>` <%= param[:required] ? "(**required**)" : "(optional)"  %><%= [" -",  param[:description]].join(" ").gsub("\n", '') unless param[:description].blank? %>
-
-<% end %>
-
-### Delete <%= service[:title] %> service
-
-Delete <%= service[:title] %> service for a project.
-
-```
-DELETE /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-### Get <%= service[:title] %> service settings
-
-Get <%= service[:title] %> service settings for a project.
-
-```
-GET /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-<% end %>
-ERB
-
-namespace :services do
-  task doc: :environment do
-    services = Service.available_services_names.map do |s|
-      service_start = Time.now
-      klass = "#{s}_service".classify.constantize
-
-      service = klass.new
-
-      service_hash = {}
-
-      service_hash[:title] = service.title
-      service_hash[:dashed_name] = s.dasherize
-      service_hash[:description] = service.description
-      service_hash[:help] = service.help
-      service_hash[:params] = service.fields.map do |p|
-        param_hash = {}
-
-        param_hash[:name] = p[:name]
-        param_hash[:description] = p[:placeholder] || p[:title]
-        param_hash[:required] = klass.validators_on(p[:name].to_sym).any? do |v|
-          v.class == ActiveRecord::Validations::PresenceValidator
-        end
-
-        param_hash
-      end
-      service_hash[:params].sort_by! { |p| p[:required] ? 0 : 1 }
-
-      puts "Collected data for: #{service.title}, #{Time.now - service_start}"
-      service_hash
-    end
-
-    doc_start = Time.now
-    doc_path = File.join(Rails.root, 'doc', 'api', 'services.md')
-
-    result = ERB.new(services_template, trim_mode: '>')
-      .result(OpenStruct.new(services: services).instance_eval { binding })
-
-    File.open(doc_path, 'w') do |f|
-      f.write result
-    end
-
-    puts "write a new service.md to: #{doc_path}, #{Time.now - doc_start}"
-  end
-end