Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/docs.gitlab-ci.yml

@@ -2,7 +2,7 @@
   extends:
     - .default-tags
     - .default-retry
-    - .only-docs-changes
+    - .only:changes-docs
   only:
     refs:
       - merge_requests
@@ -50,7 +50,7 @@ docs lint:
     - .default-tags
     - .default-retry
     - .default-only
-    - .only-docs-changes
+    - .only:changes-docs
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
   stage: test
   dependencies: []
@@ -76,7 +76,7 @@ graphql-docs-verify:
     - .default-cache
     - .default-only
     - .default-before_script
-    - .only-graphql-changes
+    - .only:changes-graphql
   variables:
     SETUP_DB: "false"
   stage: test

.gitlab/ci/frontend.gitlab-ci.yml

@@ -12,7 +12,7 @@
     - .default-only
     - .default-before_script
     - .assets-compile-cache
-    - .only-code-qa-changes
+    - .only:changes-code-backstage-qa
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1
   stage: test
   dependencies: ["setup-test-env"]
@@ -73,7 +73,7 @@ gitlab:assets:compile pull-cache:
     - .default-only
     - .default-before_script
     - .assets-compile-cache
-    - .only-code-qa-changes
+    - .only:changes-code-backstage-qa
     - .use-pg9
   stage: prepare
   script:
@@ -128,7 +128,7 @@ compile-assets pull-cache foss:
     - .default-cache
     - .default-only
     - .default-before_script
-    - .only-code-changes
+    - .only:changes-code-backstage
     - .use-pg9
   stage: test
   needs: ["setup-test-env", "compile-assets pull-cache"]
@@ -205,7 +205,7 @@ jest-foss:
     - .default-retry
     - .default-cache
     - .default-only
-    - .only-code-changes
+    - .only:changes-code-backstage
   stage: test
   dependencies: []
   cache:
@@ -238,7 +238,7 @@ webpack-dev-server:
     - .default-retry
     - .default-cache
     - .default-only
-    - .only-code-changes
+    - .only:changes-code-backstage
   stage: test
   needs: ["setup-test-env", "compile-assets pull-cache"]
   dependencies: ["setup-test-env", "compile-assets pull-cache"]

.gitlab/ci/global.gitlab-ci.yml

@@ -40,14 +40,64 @@
       - merge_requests
       - tags
-.only-code-changes:
+.code-patterns: &code-patterns
+  - ".gitlab/ci/**/*"
+  - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
+  - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
+  - ".csscomb.json"
+  - "Dockerfile.assets"
+  - "*_VERSION"
+  - "Gemfile{,.lock}"
+  - "Rakefile"
+  - "{babel.config,jest.config}.js"
+  - "config.ru"
+  - "{package.json,yarn.lock}"
+  - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
+
+.backstage-patterns: &backstage-patterns
+  - "Dangerfile"
+  - "danger/**/*"
+  - "{,ee/}fixtures/**/*"
+  - "{,ee/}rubocop/**/*"
+  - "{,ee/}spec/**/*"
+  - "doc/README.md"  # Some RSpec test rely on this file
+
+.qa-patterns: &qa-patterns
+  - ".dockerignore"
+  - "qa/**/*"
+
+.docs-patterns: &docs-patterns
+  - ".gitlab/route-map.yml"
+  - "doc/**/*"
+  - ".markdownlint.json"
+
+.graphql-patterns: &graphql-patterns
+  - "{,ee/}app/graphql/**/*"
+  - "{,ee/}lib/gitlab/graphql/**/*"
+
+.only:changes-code:
+  only:
+    changes: *code-patterns
+
+.only:changes-qa:
+  only:
+    changes: *qa-patterns
+
+.only:changes-docs:
+  only:
+    changes: *docs-patterns
+
+.only:changes-graphql:
+  only:
+    changes: *graphql-patterns
+
+.only:changes-code-backstage:
   only:
     changes:
       - ".gitlab/ci/**/*"
       - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
       - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
       - ".csscomb.json"
-      - "Dangerfile"
       - "Dockerfile.assets"
       - "*_VERSION"
       - "Gemfile{,.lock}"
@@ -55,36 +105,41 @@
       - "{babel.config,jest.config}.js"
       - "config.ru"
       - "{package.json,yarn.lock}"
-      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*"
+      - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
+      # Backstage changes
+      - "Dangerfile"
+      - "danger/**/*"
+      - "{,ee/}fixtures/**/*"
+      - "{,ee/}rubocop/**/*"
+      - "{,ee/}spec/**/*"
       - "doc/README.md"  # Some RSpec test rely on this file
-.only-qa-changes:
+.only:changes-code-qa:
   only:
     changes:
+      - ".gitlab/ci/**/*"
+      - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
+      - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
+      - ".csscomb.json"
+      - "Dockerfile.assets"
+      - "*_VERSION"
+      - "Gemfile{,.lock}"
+      - "Rakefile"
+      - "{babel.config,jest.config}.js"
+      - "config.ru"
+      - "{package.json,yarn.lock}"
+      - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
+      # QA changes
       - ".dockerignore"
       - "qa/**/*"
-.only-docs-changes:
-  only:
-    changes:
-      - ".gitlab/route-map.yml"
-      - "doc/**/*"
-      - ".markdownlint.json"
-
-.only-graphql-changes:
-  only:
-    changes:
-      - "{,ee/}app/graphql/**/*"
-      - "{,ee/}lib/gitlab/graphql/**/*"
-
-.only-code-qa-changes:
+.only:changes-code-backstage-qa:
   only:
     changes:
       - ".gitlab/ci/**/*"
       - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
       - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
       - ".csscomb.json"
-      - "Dangerfile"
       - "Dockerfile.assets"
       - "*_VERSION"
       - "Gemfile{,.lock}"
@@ -92,8 +147,15 @@
       - "{babel.config,jest.config}.js"
       - "config.ru"
       - "{package.json,yarn.lock}"
-      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*"
+      - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
+      # Backstage changes
+      - "Dangerfile"
+      - "danger/**/*"
+      - "{,ee/}fixtures/**/*"
+      - "{,ee/}rubocop/**/*"
+      - "{,ee/}spec/**/*"
       - "doc/README.md"  # Some RSpec test rely on this file
+      # QA changes
       - ".dockerignore"
       - "qa/**/*"

.gitlab/ci/memory.gitlab-ci.yml

@@ -5,7 +5,7 @@
     - .default-cache
     - .default-only
     - .default-before_script
-    - .only-code-changes
+    - .only:changes-code
 memory-static:
   extends: .only-code-memory-job-base

.gitlab/ci/pages.gitlab-ci.yml

@@ -4,7 +4,7 @@ pages:
     - .default-retry
     - .default-cache
     - .default-only
-    - .only-code-qa-changes
+    - .only:changes-code-backstage-qa
   only:
     refs:
       - master

.gitlab/ci/qa.gitlab-ci.yml

@@ -3,7 +3,7 @@
     - .default-tags
     - .default-retry
     - .default-only
-    - .only-code-qa-changes
+    - .only:changes-code-qa
   stage: test
   dependencies: []
   cache:
@@ -47,7 +47,7 @@ qa:selectors-foss:
 package-and-qa-manual:
   extends:
     - .package-and-qa-base
-    - .only-code-changes
+    - .only:changes-code
   except:
     refs:
       - master
@@ -58,7 +58,7 @@ package-and-qa-manual:
 package-and-qa:
   extends:
     - .package-and-qa-base
-    - .only-qa-changes
+    - .only:changes-qa
   except:
     refs:
       - master
@@ -69,6 +69,6 @@ package-and-qa:
 schedule:package-and-qa:
   extends:
     - .package-and-qa-base
-    - .only-code-qa-changes
+    - .only:changes-code-qa
     - .only-canonical-schedules
   needs: ["build-qa-image", "gitlab:assets:compile pull-cache"]

.gitlab/ci/rails.gitlab-ci.yml

@@ -22,7 +22,7 @@
     - .default-cache
     - .default-only
     - .default-before_script
-    - .only-code-changes
+    - .only:changes-code-backstage
 .only-code-qa-rails-job-base:
   extends:
@@ -31,7 +31,7 @@
     - .default-cache
     - .default-only
     - .default-before_script
-    - .only-code-qa-changes
+    - .only:changes-code-backstage-qa
 setup-test-env:
   extends:
@@ -251,7 +251,7 @@ static-analysis:
 downtime_check:
   extends:
     - .rake-exec
-    - .only-code-changes
+    - .only:changes-code-backstage
   except:
     refs:
       - master

.gitlab/ci/reports.gitlab-ci.yml

@@ -11,7 +11,7 @@ code_quality:
   extends:
     - .default-retry
     - .default-only
-    - .only-code-changes
+    - .only:changes-code-backstage
   stage: test
   image: docker:stable
   allow_failure: true
@@ -50,7 +50,7 @@ sast:
   extends:
     - .default-retry
     - .default-only
-    - .only-code-changes
+    - .only:changes-code-backstage-qa
   stage: test
   image: docker:stable
   variables:
@@ -132,7 +132,7 @@ dependency_scanning:
   extends:
     - .default-retry
     - .default-only
-    - .only-code-changes
+    - .only:changes-code-backstage-qa
   stage: test
   image: docker:stable
   variables:
@@ -195,7 +195,7 @@ dast:
   extends:
     - .default-retry
     - .default-only
-    - .only-code-qa-changes
+    - .only:changes-code-qa
     - .only-review
   stage: qa
   needs: ["review-deploy"]

.gitlab/ci/review.gitlab-ci.yml

@@ -23,7 +23,7 @@
 build-qa-image:
   extends:
     - .review-docker
-    - .only-code-qa-changes
+    - .only:changes-code-qa
   only:
     variables:
       - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
@@ -40,7 +40,7 @@ build-qa-image:
     - .default-tags
     - .default-retry
     - .default-only
-    - .only-code-qa-changes
+    - .only:changes-code-qa
   stage: prepare
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
   allow_failure: true
@@ -66,7 +66,7 @@ manual:review-cleanup:
 .review-build-cng-base:
   extends:
     - .default-only
-    - .only-code-qa-changes
+    - .only:changes-code-qa
     - .except-deploys
   image: ruby:2.6-alpine
   stage: review-prepare
@@ -95,7 +95,7 @@ schedule:review-build-cng:
     - .default-tags
     - .default-retry
     - .default-only
-    - .only-code-qa-changes
+    - .only:changes-code-qa
     - .except-deploys
   stage: review
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
@@ -189,7 +189,7 @@ review-cleanup-failed-deployment:
   extends:
     - .review-docker
     - .only-review
-    - .only-code-qa-changes
+    - .only:changes-code-qa
   stage: qa
   allow_failure: true
   variables:
@@ -236,7 +236,7 @@ review-qa-all:
 .review-performance-base:
   extends:
     - .review-docker
-    - .only-code-qa-changes
+    - .only:changes-code-qa
   stage: qa
   allow_failure: true
   before_script:
@@ -288,7 +288,7 @@ parallel-spec-reports:
   extends:
     - .default-tags
     - .default-only
-    - .only-code-qa-changes
+    - .only:changes-code-qa
     - .only-review
     - .except-deploys
   image: ruby:2.6-alpine

.gitlab/ci/setup.gitlab-ci.yml

@@ -6,7 +6,7 @@ cache gems:
     - .default-retry
     - .default-cache
     - .default-before_script
-    - .only-code-qa-changes
+    - .only:changes-code-backstage-qa
   stage: test
   dependencies: ["setup-test-env"]
   needs: ["setup-test-env"]
@@ -29,7 +29,7 @@ cache gems:
     - .default-tags
     - .default-retry
     - .default-only
-    - .only-code-changes
+    - .only:changes-code-backstage
   dependencies: []
 gitlab_git_test:

.gitlab/ci/test-metadata.gitlab-ci.yml

 .tests-metadata-state:
   extends:
     - .default-only
-    - .only-code-changes
+    - .only:changes-code-backstage
   variables:
     TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
   before_script:
@@ -48,7 +48,7 @@ flaky-examples-check:
     - .default-tags
     - .default-retry
     - .default-only
-    - .only-code-changes
+    - .only:changes-code-backstage
   image: ruby:2.6-alpine
   stage: post-test
   variables:

app/assets/javascripts/lib/graphql.js

@@ -26,7 +26,8 @@ export default (resolvers = {}, config = {}) => {
       createUploadLink(httpOptions),
       new BatchHttpLink(httpOptions),
     ),
-    cache: new InMemoryCache(config.cacheConfig),
+    cache: new InMemoryCache({ ...config.cacheConfig, freezeResults: true }),
     resolvers,
+    assumeImmutableResults: true,
   });
 };

app/assets/stylesheets/pages/experimental_separate_sign_up.scss

@@ -23,6 +23,7 @@
   .signup-heading h2 {
     font-weight: $gl-font-weight-bold;
+    padding: 0 10px;
     @include media-breakpoint-down(md) {
       font-size: $gl-font-size-large;

app/models/ci/build.rb

@@ -35,6 +35,10 @@ module Ci
       refspecs: -> (build) { build.merge_request_ref? }
     }.freeze
+    DEFAULT_RETRIES = {
+      scheduler_failure: 2
+    }.freeze
+
     has_one :deployment, as: :deployable, class_name: 'Deployment'
     has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
     has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id
@@ -372,18 +376,25 @@ module Ci
       pipeline.builds.retried.where(name: self.name).count
     end
-    def retries_max
-      normalized_retry.fetch(:max, 0)
+    def retry_failure?
+      max_allowed_retries = nil
+      max_allowed_retries ||= options_retry_max if retry_on_reason_or_always?
+      max_allowed_retries ||= DEFAULT_RETRIES.fetch(failure_reason.to_sym, 0)
+
+      max_allowed_retries > 0 && retries_count < max_allowed_retries
     end
-    def retry_when
-      normalized_retry.fetch(:when, ['always'])
+    def options_retry_max
+      options_retry[:max]
     end
-    def retry_failure?
-      return false if retries_max.zero? || retries_count >= retries_max
-      retry_when.include?('always') || retry_when.include?(failure_reason.to_s)
+    def options_retry_when
+      options_retry.fetch(:when, ['always'])
+    end
+    def retry_on_reason_or_always?
+      options_retry_when.include?(failure_reason.to_s) ||
+        options_retry_when.include?('always')
     end
     def latest?
@@ -831,6 +842,13 @@ module Ci
       :creating
     end
+    # Consider this object to have a structural integrity problems
+    def doom!
+      update_columns(
+        status: :failed,
+        failure_reason: :data_integrity_failure)
+    end
+
     private
     def successful_deployment_status
@@ -875,8 +893,8 @@ module Ci
     # format, but builds created before GitLab 11.5 and saved in database still
     # have the old integer only format. This method returns the retry option
     # normalized as a hash in 11.5+ format.
-    def normalized_retry
-      strong_memoize(:normalized_retry) do
+    def options_retry
+      strong_memoize(:options_retry) do
         value = options&.dig(:retry)
         value = value.is_a?(Integer) ? { max: value } : value.to_h
         value.with_indifferent_access

app/models/commit_status_enums.rb

@@ -15,7 +15,9 @@ module CommitStatusEnums
       stale_schedule: 7,
       job_execution_timeout: 8,
       archived_failure: 9,
-      unmet_prerequisites: 10
+      unmet_prerequisites: 10,
+      scheduler_failure: 11,
+      data_integrity_failure: 12
     }
   end
 end

app/presenters/commit_status_presenter.rb

@@ -11,7 +11,9 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
     stale_schedule: 'Delayed job could not be executed by some reason, please try again',
     job_execution_timeout: 'The script exceeded the maximum execution time set for the job',
     archived_failure: 'The job is archived and cannot be run',
-    unmet_prerequisites: 'The job failed to complete prerequisite tasks'
+    unmet_prerequisites: 'The job failed to complete prerequisite tasks',
+    scheduler_failure: 'The scheduler failed to assign job to the runner, please try again or contact system administrator',
+    data_integrity_failure: 'There has been a structural integrity problem detected, please contact system administrator'
   }.freeze
   private_constant :CALLOUT_FAILURE_MESSAGES
@@ -33,6 +35,6 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
   end
   def unrecoverable?
-    script_failure? || missing_dependency_failure? || archived_failure?
+    script_failure? || missing_dependency_failure? || archived_failure? || scheduler_failure? || data_integrity_failure?
   end
 end

app/services/ci/register_job_service.rb

@@ -42,26 +42,16 @@ module Ci
       end
       builds.each do |build|
-        next unless runner.can_pick?(build)
-
-        begin
-          # In case when 2 runners try to assign the same build, second runner will be declined
-          # with StateMachines::InvalidTransition or StaleObjectError when doing run! or save method.
-          if assign_runner!(build, params)
-            register_success(build)
-
-            return Result.new(build, true)
-          end
-        rescue StateMachines::InvalidTransition, ActiveRecord::StaleObjectError
-          # We are looping to find another build that is not conflicting
-          # It also indicates that this build can be picked and passed to runner.
-          # If we don't do it, basically a bunch of runners would be competing for a build
-          # and thus we will generate a lot of 409. This will increase
-          # the number of generated requests, also will reduce significantly
-          # how many builds can be picked by runner in a unit of time.
-          # In case we hit the concurrency-access lock,
-          # we still have to return 409 in the end,
-          # to make sure that this is properly handled by runner.
+        result = process_build(build, params)
+        next unless result
+
+        if result.valid?
+          register_success(result.build)
+
+          return result
+        else
+          # The usage of valid: is described in
+          # handling of ActiveRecord::StaleObjectError
           valid = false
         end
       end
@@ -73,6 +63,35 @@ module Ci
     private
+    def process_build(build, params)
+      return unless runner.can_pick?(build)
+
+      # In case when 2 runners try to assign the same build, second runner will be declined
+      # with StateMachines::InvalidTransition or StaleObjectError when doing run! or save method.
+      if assign_runner!(build, params)
+        Result.new(build, true)
+      end
+    rescue StateMachines::InvalidTransition, ActiveRecord::StaleObjectError
+      # We are looping to find another build that is not conflicting
+      # It also indicates that this build can be picked and passed to runner.
+      # If we don't do it, basically a bunch of runners would be competing for a build
+      # and thus we will generate a lot of 409. This will increase
+      # the number of generated requests, also will reduce significantly
+      # how many builds can be picked by runner in a unit of time.
+      # In case we hit the concurrency-access lock,
+      # we still have to return 409 in the end,
+      # to make sure that this is properly handled by runner.
+      Result.new(nil, false)
+    rescue => ex
+      raise ex unless Feature.enabled?(:ci_doom_build, default_enabled: true)
+
+      scheduler_failure!(build)
+      track_exception_for_build(ex, build)
+
+      # skip, and move to next one
+      nil
+    end
+
     def assign_runner!(build, params)
       build.runner_id = runner.id
       build.runner_session_attributes = params[:session] if params[:session].present?
@@ -96,6 +115,28 @@ module Ci
       true
     end
+    def scheduler_failure!(build)
+      Gitlab::OptimisticLocking.retry_lock(build, 3) do |subject|
+        subject.drop!(:scheduler_failure)
+      end
+    rescue => ex
+      build.doom!
+
+      # This requires extra exception, otherwise we would loose information
+      # why we cannot perform `scheduler_failure`
+      track_exception_for_build(ex, build)
+    end
+
+    def track_exception_for_build(ex, build)
+      Gitlab::Sentry.track_acceptable_exception(ex, extra: {
+        build_id: build.id,
+        build_name: build.name,
+        build_stage: build.stage,
+        pipeline_id: build.pipeline_id,
+        project_id: build.project_id
+      })
+    end
+
     # rubocop: disable CodeReuse/ActiveRecord
     def builds_for_shared_runner
       new_builds.

app/views/registrations/welcome.html.haml

-- content_for(:page_title, _('Welcome to GitLab<br>%{username}!' % { username: html_escape(current_user.username) }).html_safe)
+- content_for(:page_title, _('Welcome to GitLab %{username}!') % { username: current_user.username })
 - max_name_length = 128
 .text-center.mb-3
   = _('In order to tailor your experience with GitLab<br>we would like to know a bit more about you.').html_safe

changelogs/unreleased/13539-license-compliance-approval-required.yml

+---
+title: Show approval required status in license compliance
+merge_request: 19114
+author:
+type: changed

changelogs/unreleased/make-register-job-service-to-be-resillient.yml

+---
+title: Make `jobs/request` to be resillient
+merge_request: 19150
+author:
+type: fixed