Add latest changes from gitlab-org/gitlab@master

app/services/ci/pipeline_processing/legacy_processing_service.rb

+# frozen_string_literal: true
+
+module Ci
+  module PipelineProcessing
+    class LegacyProcessingService
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :pipeline
+
+      def initialize(pipeline)
+        @pipeline = pipeline
+      end
+
+      def execute(trigger_build_ids = nil)
+        success = process_stages_without_needs
+
+        # we evaluate dependent needs,
+        # only when the another job has finished
+        success = process_builds_with_needs(trigger_build_ids) || success
+
+        @pipeline.update_status
+
+        success
+      end
+
+      private
+
+      def process_stages_without_needs
+        stage_indexes_of_created_processables_without_needs.flat_map do |index|
+          process_stage_without_needs(index)
+        end.any?
+      end
+
+      def process_stage_without_needs(index)
+        current_status = status_for_prior_stages(index)
+
+        return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
+
+        created_processables_in_stage_without_needs(index).find_each.select do |build|
+          process_build(build, current_status)
+        end.any?
+      end
+
+      def process_builds_with_needs(trigger_build_ids)
+        return false unless trigger_build_ids.present?
+        return false unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
+
+        # we find processables that are dependent:
+        # 1. because of current dependency,
+        trigger_build_names = pipeline.processables.latest
+          .for_ids(trigger_build_ids).names
+
+        # 2. does not have builds that not yet complete
+        incomplete_build_names = pipeline.processables.latest
+          .incomplete.names
+
+        # Each found processable is guaranteed here to have completed status
+        created_processables
+          .with_needs(trigger_build_names)
+          .without_needs(incomplete_build_names)
+          .find_each
+          .map(&method(:process_build_with_needs))
+          .any?
+      end
+
+      def process_build_with_needs(build)
+        current_status = status_for_build_needs(build.needs.map(&:name))
+
+        return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
+
+        process_build(build, current_status)
+      end
+
+      def process_build(build, current_status)
+        Gitlab::OptimisticLocking.retry_lock(build) do |subject|
+          Ci::ProcessBuildService.new(project, subject.user)
+            .execute(subject, current_status)
+        end
+      end
+
+      def status_for_prior_stages(index)
+        pipeline.processables.status_for_prior_stages(index)
+      end
+
+      def status_for_build_needs(needs)
+        pipeline.processables.status_for_names(needs)
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def stage_indexes_of_created_processables_without_needs
+        created_processables_without_needs.order(:stage_idx)
+          .pluck(Arel.sql('DISTINCT stage_idx'))
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def created_processables_in_stage_without_needs(index)
+        created_processables_without_needs
+          .with_preloads
+          .for_stage(index)
+      end
+
+      def created_processables_without_needs
+        if Feature.enabled?(:ci_dag_support, project, default_enabled: true)
+          pipeline.processables.created.without_needs
+        else
+          pipeline.processables.created
+        end
+      end
+
+      def created_processables
+        pipeline.processables.created
+      end
+
+      def project
+        pipeline.project
+      end
+    end
+  end
+end

app/services/ci/process_pipeline_service.rb

@@ -2,8 +2,6 @@
 module Ci
   class ProcessPipelineService
-    include Gitlab::Utils::StrongMemoize
-
     attr_reader :pipeline
     def initialize(pipeline)
@@ -13,105 +11,13 @@ module Ci
     def execute(trigger_build_ids = nil)
       update_retried
-      success = process_stages_without_needs
-
-      # we evaluate dependent needs,
-      # only when the another job has finished
-      success = process_builds_with_needs(trigger_build_ids) || success
-
-      @pipeline.update_status
-
-      success
+      Ci::PipelineProcessing::LegacyProcessingService
+        .new(pipeline)
+        .execute(trigger_build_ids)
     end
     private
-    def process_stages_without_needs
-      stage_indexes_of_created_processables_without_needs.flat_map do |index|
-        process_stage_without_needs(index)
-      end.any?
-    end
-
-    def process_stage_without_needs(index)
-      current_status = status_for_prior_stages(index)
-
-      return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
-
-      created_processables_in_stage_without_needs(index).find_each.select do |build|
-        process_build(build, current_status)
-      end.any?
-    end
-
-    def process_builds_with_needs(trigger_build_ids)
-      return false unless trigger_build_ids.present?
-      return false unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
-
-      # we find processables that are dependent:
-      # 1. because of current dependency,
-      trigger_build_names = pipeline.processables.latest
-        .for_ids(trigger_build_ids).names
-
-      # 2. does not have builds that not yet complete
-      incomplete_build_names = pipeline.processables.latest
-        .incomplete.names
-
-      # Each found processable is guaranteed here to have completed status
-      created_processables
-        .with_needs(trigger_build_names)
-        .without_needs(incomplete_build_names)
-        .find_each
-        .map(&method(:process_build_with_needs))
-        .any?
-    end
-
-    def process_build_with_needs(build)
-      current_status = status_for_build_needs(build.needs.map(&:name))
-
-      return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
-
-      process_build(build, current_status)
-    end
-
-    def process_build(build, current_status)
-      Gitlab::OptimisticLocking.retry_lock(build) do |subject|
-        Ci::ProcessBuildService.new(project, build.user)
-          .execute(subject, current_status)
-      end
-    end
-
-    def status_for_prior_stages(index)
-      pipeline.processables.status_for_prior_stages(index)
-    end
-
-    def status_for_build_needs(needs)
-      pipeline.processables.status_for_names(needs)
-    end
-
-    # rubocop: disable CodeReuse/ActiveRecord
-    def stage_indexes_of_created_processables_without_needs
-      created_processables_without_needs.order(:stage_idx)
-        .pluck(Arel.sql('DISTINCT stage_idx'))
-    end
-    # rubocop: enable CodeReuse/ActiveRecord
-
-    def created_processables_in_stage_without_needs(index)
-      created_processables_without_needs
-        .with_preloads
-        .for_stage(index)
-    end
-
-    def created_processables_without_needs
-      if Feature.enabled?(:ci_dag_support, project, default_enabled: true)
-        pipeline.processables.created.without_needs
-      else
-        pipeline.processables.created
-      end
-    end
-
-    def created_processables
-      pipeline.processables.created
-    end
-
     # This method is for compatibility and data consistency and should be removed with 9.3 version of GitLab
     # This replicates what is db/post_migrate/20170416103934_upate_retried_for_ci_build.rb
     # and ensures that functionality will not be broken before migration is run
@@ -131,9 +37,5 @@ module Ci
         .update_all(retried: true) if latest_statuses.any?
     end
     # rubocop: enable CodeReuse/ActiveRecord
-
-    def project
-      pipeline.project
-    end
   end
 end

doc/development/gitaly.md

@@ -207,6 +207,21 @@ To use a custom Gitaly repository in CI, for instance if you want your
 GitLab fork to always use your own Gitaly fork, set `GITALY_REPO_URL`
 as a [CI environment variable](../ci/variables/README.md#gitlab-cicd-environment-variables).
+### Use a locally modified version of Gitaly RPC client
+
+If you are making changes to the RPC client, such as adding a new endpoint or adding a new
+parameter to an existing endpoint, follow the guide for
+[Gitaly proto](https://gitlab.com/gitlab-org/gitaly/blob/master/proto/README.md). After pushing
+the branch with the changes (`new-feature-branch`, for example):
+
+1. Change the `gitaly` line in the Rails' `Gemfile` to:
+
+   ```ruby
+   gem 'gitaly', git: 'https://gitlab.com/gitlab-org/gitaly.git', branch: 'new-feature-branch'
+   ```
+
+1. Run `bundle install` to use the modified RPC client.
+
 ---
 [Return to Development documentation](README.md)

package.json

@@ -22,8 +22,8 @@
     "prettier-staged-save": "node ./scripts/frontend/prettier.js save",
     "prettier-all": "node ./scripts/frontend/prettier.js check-all",
     "prettier-all-save": "node ./scripts/frontend/prettier.js save-all",
-    "stylelint": "node node_modules/stylelint/bin/stylelint.js app/assets/stylesheets/**/*.*  ee/app/assets/stylesheets/**/*.* !**/vendors/**",
-    "stylelint-file": "node node_modules/stylelint/bin/stylelint.js",
+    "stylelint": "yarn stylelint-file app/assets/stylesheets/**/*.*  ee/app/assets/stylesheets/**/*.* !**/vendors/**",
+    "stylelint-file": "BROWSERSLIST_IGNORE_OLD_DATA=true node node_modules/stylelint/bin/stylelint.js",
     "stylelint-create-utility-map": "node scripts/frontend/stylelint/stylelint-utility-map.js",
     "test": "node scripts/frontend/test",
     "webpack": "NODE_OPTIONS=\"--max-old-space-size=3584\" webpack --config config/webpack.config.js",
@@ -39,8 +39,8 @@
     "@babel/plugin-syntax-dynamic-import": "^7.2.0",
     "@babel/plugin-syntax-import-meta": "^7.2.0",
     "@babel/preset-env": "^7.6.2",
-    "@gitlab/svgs": "^1.88.0",
-    "@gitlab/ui": "8.10.0",
+    "@gitlab/svgs": "^1.89.0",
+    "@gitlab/ui": "8.15.0",
     "@gitlab/visual-review-tools": "1.5.1",
     "@sentry/browser": "^5.10.2",
     "@sourcegraph/code-host-integration": "^0.0.18",

qa/qa/specs/features/browser_ui/non_devops/performance_bar_spec.rb

@@ -2,7 +2,7 @@
 module QA
   context 'Non-devops' do
-    describe 'Performance bar display', :requires_admin, quarantine: 'https://gitlab.com/gitlab-org/gitlab/issues/196141' do
+    describe 'Performance bar display', :requires_admin do
       context 'when logged in as an admin user' do
         # 4 metrics: pg, gitaly, redis, total
         let(:metrics_count) { 4 }

spec/fixtures/lib/gitlab/import_export/complex/project.json

@@ -2002,7 +2002,7 @@
     },
     {
       "id": 31,
-      "title": "Libero nam magnam incidunt eaque placeat error et.",
+      "title": "issue_with_timelogs",
       "author_id": 16,
       "project_id": 5,
       "created_at": "2016-06-14T15:02:07.280Z",
@@ -2016,6 +2016,16 @@
       "confidential": false,
       "due_date": null,
       "moved_to_id": null,
+      "timelogs": [
+        {
+          "id": 1,
+          "time_spent": 72000,
+          "user_id": 1,
+          "created_at": "2019-12-27T09:15:22.302Z",
+          "updated_at": "2019-12-27T09:15:22.302Z",
+          "spent_at": "2019-12-27T00:00:00.000Z"
+        }
+      ],
       "notes": [
         {
           "id": 423,
@@ -2347,7 +2357,30 @@
       ]
     }
   ],
-  "releases": [],
+  "releases": [
+    {
+      "id": 1,
+      "tag": "release-1.1",
+      "description": "Some release notes",
+      "project_id": 5,
+      "created_at": "2019-12-26T10:17:14.621Z",
+      "updated_at": "2019-12-26T10:17:14.621Z",
+      "author_id": 1,
+      "name": "release-1.1",
+      "sha": "901de3a8bd5573f4a049b1457d28bc1592ba6bf9",
+      "released_at": "2019-12-26T10:17:14.615Z",
+      "links": [
+        {
+          "id": 1,
+          "release_id" : 1,
+          "url": "http://localhost/namespace6/project6/-/jobs/140463678/artifacts/download",
+          "name": "release-1.1.dmg",
+          "created_at": "2019-12-26T10:17:14.621Z",
+          "updated_at": "2019-12-26T10:17:14.621Z"
+        }
+      ]
+    }
+  ],
   "project_members": [
     {
       "id": 36,
@@ -6816,6 +6849,40 @@
       "duration": null,
       "stages": [
       ]
+    },
+    {
+      "id": 42,
+      "project_id": 5,
+      "ref": "master",
+      "sha": "ce84140e8b878ce6e7c4d298c7202ff38170e3ac",
+      "before_sha": null,
+      "push_data": null,
+      "created_at": "2016-03-22T15:20:35.763Z",
+      "updated_at": "2016-03-22T15:20:35.763Z",
+      "tag": false,
+      "yaml_errors": null,
+      "committed_at": null,
+      "status": "failed",
+      "started_at": null,
+      "finished_at": null,
+      "duration": null,
+      "stages": [
+      ],
+      "source": "external_pull_request_event",
+      "external_pull_request":
+      {
+        "id": 3,
+        "pull_request_iid": 4,
+        "source_branch": "feature",
+        "target_branch": "master",
+        "source_repository": "the-repository",
+        "target_repository": "the-repository",
+        "source_sha": "ce84140e8b878ce6e7c4d298c7202ff38170e3ac",
+        "target_sha": "a09386439ca39abe575675ffd4b89ae824fec22f",
+        "status": "open",
+	"created_at": "2016-03-22T15:20:35.763Z",
+	"updated_at": "2016-03-22T15:20:35.763Z"
+      }
     }
   ],
   "triggers": [
@@ -6835,6 +6902,21 @@
       "updated_at": "2017-01-16T15:25:29.637Z"
     }
   ],
+  "pipeline_schedules": [
+    {
+      "id": 1,
+      "description": "Schedule Description",
+      "ref": "master",
+      "cron": "0 4 * * 0",
+      "cron_timezone": "UTC",
+      "next_run_at": "2019-12-29T04:19:00.000Z",
+      "project_id": 5,
+      "owner_id": 1,
+      "active": true,
+      "created_at": "2019-12-26T10:14:57.778Z",
+      "updated_at": "2019-12-26T10:14:57.778Z"
+    }
+  ],
   "container_expiration_policy": {
     "created_at": "2019-12-13 13:45:04 UTC",
     "updated_at": "2019-12-13 13:45:04 UTC",
@@ -7354,6 +7436,33 @@
   "ci_cd_settings": {
     "group_runners_enabled": false
   },
+  "auto_devops": {
+    "id": 1,
+    "created_at": "2017-10-19T15:36:23.466Z",
+    "updated_at": "2017-10-19T15:36:23.466Z",
+    "enabled": null,
+    "deploy_strategy": "continuous"
+  },
+  "error_tracking_setting": {
+    "api_url": "https://gitlab.example.com/api/0/projects/sentry-org/sentry-project",
+    "project_name": "Sentry Project",
+    "organization_name": "Sentry Org"
+  },
+  "external_pull_requests": [
+    {
+       "id": 3,
+       "pull_request_iid": 4,
+       "source_branch": "feature",
+       "target_branch": "master",
+       "source_repository": "the-repository",
+       "target_repository": "the-repository",
+       "source_sha": "ce84140e8b878ce6e7c4d298c7202ff38170e3ac",
+       "target_sha": "a09386439ca39abe575675ffd4b89ae824fec22f",
+       "status": "open",
+       "created_at": "2019-12-24T14:04:50.053Z",
+       "updated_at": "2019-12-24T14:05:18.138Z"
+    }
+  ],
   "boards": [
     {
       "id": 29,

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -116,6 +116,15 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         expect(Issue.find_by(title: 'Issue without assignees').assignees).to be_empty
       end
+      it 'restores timelogs for issues' do
+        timelog = Issue.find_by(title: 'issue_with_timelogs').timelogs.last
+
+        aggregate_failures do
+          expect(timelog.time_spent).to eq(72000)
+          expect(timelog.spent_at).to eq("2019-12-27T00:00:00.000Z")
+        end
+      end
+
       it 'contains the merge access levels on a protected branch' do
         expect(ProtectedBranch.first.merge_access_levels).not_to be_empty
       end
@@ -229,6 +238,11 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         expect(@project.ci_cd_settings.group_runners_enabled?).to eq(false)
       end
+      it 'restores `auto_devops`' do
+        expect(@project.auto_devops_enabled?).to eq(true)
+        expect(@project.auto_devops.deploy_strategy).to eq('continuous')
+      end
+
       it 'restores the correct service' do
         expect(CustomIssueTrackerService.first).not_to be_nil
       end
@@ -268,6 +282,55 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         end
       end
+      it 'restores error_tracking_setting' do
+        setting = @project.error_tracking_setting
+
+        aggregate_failures do
+          expect(setting.api_url).to eq("https://gitlab.example.com/api/0/projects/sentry-org/sentry-project")
+          expect(setting.project_name).to eq("Sentry Project")
+          expect(setting.organization_name).to eq("Sentry Org")
+        end
+      end
+
+      it 'restores external pull requests' do
+        external_pr = @project.external_pull_requests.last
+
+        aggregate_failures do
+          expect(external_pr.pull_request_iid).to eq(4)
+          expect(external_pr.source_branch).to eq("feature")
+          expect(external_pr.target_branch).to eq("master")
+          expect(external_pr.status).to eq("open")
+        end
+      end
+
+      it 'restores pipeline schedules' do
+        pipeline_schedule = @project.pipeline_schedules.last
+
+        aggregate_failures do
+          expect(pipeline_schedule.description).to eq('Schedule Description')
+          expect(pipeline_schedule.ref).to eq('master')
+          expect(pipeline_schedule.cron).to eq('0 4 * * 0')
+          expect(pipeline_schedule.cron_timezone).to eq('UTC')
+          expect(pipeline_schedule.active).to eq(true)
+        end
+      end
+
+      it 'restores releases with links' do
+        release = @project.releases.last
+        link = release.links.last
+
+        aggregate_failures do
+          expect(release.tag).to eq('release-1.1')
+          expect(release.description).to eq('Some release notes')
+          expect(release.name).to eq('release-1.1')
+          expect(release.sha).to eq('901de3a8bd5573f4a049b1457d28bc1592ba6bf9')
+          expect(release.released_at).to eq('2019-12-26T10:17:14.615Z')
+
+          expect(link.url).to eq('http://localhost/namespace6/project6/-/jobs/140463678/artifacts/download')
+          expect(link.name).to eq('release-1.1.dmg')
+        end
+      end
+
       context 'Merge requests' do
         it 'always has the new project as a target' do
           expect(MergeRequest.find_by_title('MR1').target_project).to eq(@project)
@@ -321,9 +384,9 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         end
         it 'has the correct number of pipelines and statuses' do
-          expect(@project.ci_pipelines.size).to eq(6)
-          @project.ci_pipelines.order(:id).zip([2, 2, 2, 2, 2, 0])
+          expect(@project.ci_pipelines.size).to eq(7)
+          @project.ci_pipelines.order(:id).zip([2, 2, 2, 2, 2, 0, 0])
             .each do |(pipeline, expected_status_size)|
             expect(pipeline.statuses.size).to eq(expected_status_size)
           end
@@ -332,7 +395,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
       context 'when restoring hierarchy of pipeline, stages and jobs' do
         it 'restores pipelines' do
-          expect(Ci::Pipeline.all.count).to be 6
+          expect(Ci::Pipeline.all.count).to be 7
         end
         it 'restores pipeline stages' do
@@ -358,6 +421,12 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         it 'restores a Hash for CommitStatus options' do
           expect(CommitStatus.all.map(&:options).compact).to all(be_a(Hash))
         end
+
+        it 'restores external pull request for the restored pipeline' do
+          pipeline_with_external_pr = @project.ci_pipelines.order(:id).last
+
+          expect(pipeline_with_external_pr.external_pull_request).to be_persisted
+        end
       end
     end
   end

spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_relative 'shared_processing_service.rb'
+
+describe Ci::PipelineProcessing::LegacyProcessingService do
+  it_behaves_like 'Pipeline Processing Service'
+end

yarn.lock

@@ -732,15 +732,15 @@
   dependencies:
     vue-eslint-parser "^6.0.4"
-"@gitlab/svgs@^1.88.0":
-  version "1.88.0"
-  resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.88.0.tgz#0a9b72e9591264fcac592ebf9944665c70f48de2"
-  integrity sha512-ZgepCvZoB/lFdgttHtu8+9YlRZlVc9MnHDbbqcQCFBvrfOjY1wq12ikxnNbwKj8QNA47TRJvSS0TkHgMWYnbsA==
-"@gitlab/ui@8.10.0":
-  version "8.10.0"
-  resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-8.10.0.tgz#885ea8fe695ccff859821bd4ad4cefbea086e3b6"
-  integrity sha512-bbq+7iiptsNUWtPBQs/ek5uDnkSQ7QGzJwddfZSvQPMuq+50fDSy3Gab1cwvZtSj1fSvAny0ksqbpQl1w+5AgA==
+"@gitlab/svgs@^1.89.0":
+  version "1.89.0"
+  resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.89.0.tgz#5bdaff1b0af1cc07ed34e89c21c34c7c6a3e1caa"
+  integrity sha512-vI6VobZs6mq2Bbiej5bYMHyvtn8kD1O/uHSlyY9jgJoa2TXU+jFI9DqUpJmx8EIHt+o0qm/8G3XsFGEr5gLb7Q==
+"@gitlab/ui@8.15.0":
+  version "8.15.0"
+  resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-8.15.0.tgz#51fa3f2b4ccb8454bcb9680acb334bc88fe15f3d"
+  integrity sha512-M9hnLVRMUF5DDfwPtR5CLsCyiWgjslqg2p37a6qwjdjZ+ST5t0Vr/44Mg4Lz4y2zxqjDaSmR4KtmipvykeQx1A==
   dependencies:
     "@babel/standalone" "^7.0.0"
     "@gitlab/vue-toasted" "^1.3.0"
@@ -750,6 +750,7 @@
     highlight.js "^9.13.1"
     js-beautify "^1.8.8"
     lodash "^4.17.14"
+    portal-vue "^2.1.6"
     resize-observer-polyfill "^1.5.1"
     url-search-params-polyfill "^5.0.0"
     vue "^2.6.10"
@@ -8742,10 +8743,10 @@ popper.js@^1.14.7, popper.js@^1.15.0:
   resolved "https://registry.yarnpkg.com/popper.js/-/popper.js-1.15.0.tgz#5560b99bbad7647e9faa475c6b8056621f5a4ff2"
   integrity sha512-w010cY1oCUmI+9KwwlWki+r5jxKfTFDVoadl7MSrIujHU5MJ5OR6HTDj6Xo8aoR/QsA56x8jKjA59qGH4ELtrA==
-portal-vue@^2.1.5:
-  version "2.1.5"
-  resolved "https://registry.yarnpkg.com/portal-vue/-/portal-vue-2.1.5.tgz#ecd0997cb32958205151cb72f40fd4f38d175e5c"
-  integrity sha512-vZmdMn0mOo7puvxoMQ5zju6S29aFD+9yygJxyWQtPaMXS9xunAeoYdnx6yzfL9J8HD8pMZYgSieEIbioAKhrSQ==
+portal-vue@^2.1.5, portal-vue@^2.1.6:
+  version "2.1.7"
+  resolved "https://registry.yarnpkg.com/portal-vue/-/portal-vue-2.1.7.tgz#ea08069b25b640ca08a5b86f67c612f15f4e4ad4"
+  integrity sha512-+yCno2oB3xA7irTt0EU5Ezw22L2J51uKAacE/6hMPMoO/mx3h4rXFkkBkT4GFsMDv/vEe8TNKC3ujJJ0PTwb6g==
 portfinder@^1.0.24:
   version "1.0.24"