Add latest changes from gitlab-org/gitlab@master

changelogs/unreleased/37403-npm-install-fails-with-ci_job_token-in-npmrc.yml

+---
+title: Allow NPM package downloads with CI_JOB_TOKEN
+merge_request: 20868
+author:
+type: added

changelogs/unreleased/Delete-board_service-js-in-app-folder.yml

+---
+title: delete board_service.js
+merge_request: 20168
+author: nuwe1
+type: other

changelogs/unreleased/add-root-ci-config-including-user-defined-config.yml

+---
+title: Allow CI config path to point to a URL or file in a different repository
+merge_request: 20179
+author:
+type: added

changelogs/unreleased/dz-move-wiki-route.yml

+---
+title: Move wiki routing under /-/ scope
+merge_request: 21185
+author:
+type: deprecated

changelogs/unreleased/fj-38068-fix-snippet-route-refactor.yml

+---
+title: Fix snippet routes
+merge_request: 21248
+author:
+type: fixed

changelogs/unreleased/ph-31406-fetchWidgetDataAsync.yml

+---
+title: Fetches initial merge request widget data async
+merge_request: 20719
+author:
+type: changed

config/routes/project.rb

@@ -258,6 +258,10 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
             post :list_projects
           end
         end
+
+        # The wiki routing contains wildcard characters so
+        # its preferable to keep it below all other project routes
+        draw :wiki
       end
       # End of the /-/ scope.
@@ -523,9 +527,8 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         post :web_ide_clientside_preview
       end
-      # Since both wiki and repository routing contains wildcard characters
+      # The repository routing contains wildcard characters so
       # its preferable to keep it below all other project routes
-      draw :wiki
       draw :repository
       # All new routes should go under /-/ scope.
@@ -542,7 +545,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
                                             :forks, :group_links, :import, :avatar, :mirror,
                                             :cycle_analytics, :mattermost, :variables, :triggers,
                                             :environments, :protected_environments, :error_tracking,
-                                            :serverless, :clusters, :audit_events)
+                                            :serverless, :clusters, :audit_events, :wikis)
     end
     # rubocop: disable Cop/PutProjectRoutesUnderScope

config/webpack.config.js

@@ -185,7 +185,7 @@ module.exports = {
         options: { limit: 2048 },
       },
       {
-        test: /\_worker\.js$/,
+        test: /_worker\.js$/,
         use: [
           {
             loader: 'worker-loader',

db/post_migrate/20191108202723_add_unique_constraint_to_software_licenses.rb

+# frozen_string_literal: true
+
+class AddUniqueConstraintToSoftwareLicenses < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+  DOWNTIME = false
+  NEW_INDEX = 'index_software_licenses_on_unique_name'
+  OLD_INDEX = 'index_software_licenses_on_name'
+
+  disable_ddl_transaction!
+
+  # 12 software licenses will be removed on GitLab.com
+  # 0 software license policies will be updated on GitLab.com
+  def up(attempts: 100)
+    remove_redundant_software_licenses!
+
+    add_concurrent_index :software_licenses, :name, unique: true, name: NEW_INDEX
+    remove_concurrent_index :software_licenses, :name, name: OLD_INDEX
+  rescue ActiveRecord::RecordNotUnique
+    retry if (attempts -= 1) > 0
+
+    raise StandardError, <<~EOS
+      Failed to add an unique index to software_licenses, despite retrying the
+      migration 100 times.
+
+      See https://gitlab.com/gitlab-org/gitlab/merge_requests/19840.
+    EOS
+  end
+
+  def down
+    remove_concurrent_index :software_licenses, :name, unique: true, name: NEW_INDEX
+    add_concurrent_index :software_licenses, :name, name: OLD_INDEX
+  end
+
+  private
+
+  def remove_redundant_software_licenses!
+    redundant_software_licenses = execute <<~SQL
+      SELECT min(id) id, name
+      FROM software_licenses
+      WHERE name IN (select name from software_licenses group by name having count(name) > 1)
+      GROUP BY name
+    SQL
+    say "Detected #{redundant_software_licenses.count} duplicates."
+
+    redundant_software_licenses.each_row do |id, name|
+      say_with_time("Reassigning policies that reference software license #{name}.") do
+        duplicates = software_licenses.where.not(id: id).where(name: name)
+
+        software_license_policies
+          .where(software_license_id: duplicates)
+          .update_all(software_license_id: id)
+
+        duplicates.delete_all
+      end
+    end
+  end
+
+  def table(name)
+    Class.new(ActiveRecord::Base) { self.table_name = name }
+  end
+
+  def software_licenses
+    @software_licenses ||= table(:software_licenses)
+  end
+
+  def software_license_policies
+    @software_license_policies ||= table(:software_license_policies)
+  end
+end

db/schema.rb

@@ -3698,7 +3698,7 @@ ActiveRecord::Schema.define(version: 2019_12_02_031812) do
   create_table "software_licenses", id: :serial, force: :cascade do |t|
     t.string "name", null: false
     t.string "spdx_identifier", limit: 255
-    t.index ["name"], name: "index_software_licenses_on_name"
+    t.index ["name"], name: "index_software_licenses_on_unique_name", unique: true
     t.index ["spdx_identifier"], name: "index_software_licenses_on_spdx_identifier"
   end

doc/user/project/integrations/webhooks.md

@@ -969,7 +969,7 @@ X-Gitlab-Event: Wiki Page Hook
     "http_url": "http://example.com/root/awesome-project.git"
   },
   "wiki": {
-    "web_url": "http://example.com/root/awesome-project/wikis/home",
+    "web_url": "http://example.com/root/awesome-project/-/wikis/home",
     "git_ssh_url": "git@example.com:root/awesome-project.wiki.git",
     "git_http_url": "http://example.com/root/awesome-project.wiki.git",
     "path_with_namespace": "root/awesome-project.wiki",
@@ -981,7 +981,7 @@ X-Gitlab-Event: Wiki Page Hook
     "format": "markdown",
     "message": "adding an awesome page to the wiki",
     "slug": "awesome",
-    "url": "http://example.com/root/awesome-project/wikis/awesome",
+    "url": "http://example.com/root/awesome-project/-/wikis/awesome",
     "action": "create"
   }
 }

doc/user/project/pipelines/settings.md

@@ -67,20 +67,37 @@ For information about setting a maximum artifact size for a project, see
 ## Custom CI configuration path
-> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/12509) in GitLab 9.4.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/12509) in GitLab 9.4.
+> - [Support for external `.gitlab-ci.yml` locations](https://gitlab.com/gitlab-org/gitlab/issues/14376) introduced in GitLab 12.6.
 By default we look for the `.gitlab-ci.yml` file in the project's root
-directory. If you require a different location **within** the repository,
-you can set a custom path that will be used to look up the configuration file,
-this path should be **relative** to the root.
-Here are some valid examples:
-- `.gitlab-ci.yml`
+directory. If needed, you can specify an alternate path and file name, including locations outside the project.
+Hosting the configuration file in a separate project will allow stricter control of the
+configuration file. You can limit access to the project hosting the configuration to only people
+with proper authorization, and users can use the configuration for their pipelines,
+without being able to modify it.
+If the CI configuration will stay within the repository, but in a
+location different than the default,
+the path must be relative to the root directory. Examples of valid paths and file names:
+
+- `.gitlab-ci.yml` (default)
 - `.my-custom-file.yml`
 - `my/path/.gitlab-ci.yml`
 - `my/path/.my-custom-file.yml`
+If the CI configuration will be hosted in a different project within GitLab, the path must be relative
+to the root directory in the other project, with the group and project name added to the end:
+
+- `.gitlab-ci.yml@mygroup/another-project`
+- `my/path/.my-custom-file.yml@mygroup/another-project`
+
+If the CI configuration will be hosted on an external site, different than the GitLab instance,
+the URL link must end with `.yml`:
+
+- `http://example.com/generate/ci/config.yml`
+
 The path can be customized at a project level. To customize the path:
 1. Go to the project's **Settings > CI / CD**.

lib/gitlab/ci/pipeline/chain/config/content.rb

@@ -8,21 +8,28 @@ module Gitlab
           class Content < Chain::Base
             include Chain::Helpers
-            def perform!
-              return if @command.config_content
-
-              if content = content_from_repo
-                @command.config_content = content
-                @pipeline.config_source = :repository_source
-                # TODO: we should persist ci_config_path
-                # @pipeline.config_path = ci_config_path
-              elsif content = content_from_auto_devops
-                @command.config_content = content
-                @pipeline.config_source = :auto_devops_source
-              end
-              unless @command.config_content
-                return error("Missing #{ci_config_path} file")
+            SOURCES = [
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Repository,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Remote,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops
+            ].freeze
+
+            LEGACY_SOURCES = [
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyRepository,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyAutoDevops
+            ].freeze
+            def perform!
+              if config = find_config
+                # TODO: we should persist config_content
+                # @pipeline.config_content = config.content
+                @command.config_content = config.content
+                @pipeline.config_source = config.source
+              else
+                error('Missing CI config file')
               end
             end
@@ -32,24 +39,21 @@ module Gitlab
             private
-            def content_from_repo
-              return unless project
-              return unless @pipeline.sha
-              return unless ci_config_path
-              project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
-            rescue GRPC::NotFound, GRPC::Internal
+            def find_config
+              sources.each do |source|
+                config = source.new(@pipeline, @command)
+                return config if config.exists?
+              end
               nil
             end
-            def content_from_auto_devops
-              return unless project&.auto_devops_enabled?
-
-              Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
-            end
-
-            def ci_config_path
-              project.ci_config_path.presence || '.gitlab-ci.yml'
+            def sources
+              if Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true)
+                SOURCES
+              else
+                LEGACY_SOURCES
+              end
             end
           end
         end

lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class AutoDevops < Source
+              def content
+                strong_memoize(:content) do
+                  next unless project&.auto_devops_enabled?
+
+                  template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+                  YAML.dump('include' => [{ 'template' => template.full_name }])
+                end
+              end
+
+              def source
+                :auto_devops_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/external_project.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class ExternalProject < Source
+              def content
+                strong_memoize(:content) do
+                  next unless external_project_path?
+
+                  path_file, path_project = ci_config_path.split('@', 2)
+                  YAML.dump('include' => [{ 'project' => path_project, 'file' => path_file }])
+                end
+              end
+
+              def source
+                :external_project_source
+              end
+
+              private
+
+              # Example: path/to/.gitlab-ci.yml@another-group/another-project
+              def external_project_path?
+                ci_config_path =~ /\A.+(yml|yaml)@.+\z/
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class LegacyAutoDevops < Source
+              def content
+                strong_memoize(:content) do
+                  next unless project&.auto_devops_enabled?
+
+                  template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+                  template.content
+                end
+              end
+
+              def source
+                :auto_devops_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/legacy_repository.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class LegacyRepository < Source
+              def content
+                strong_memoize(:content) do
+                  next unless project
+                  next unless @pipeline.sha
+                  next unless ci_config_path
+
+                  project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
+                rescue GRPC::NotFound, GRPC::Internal
+                  nil
+                end
+              end
+
+              def source
+                :repository_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/remote.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class Remote < Source
+              def content
+                strong_memoize(:content) do
+                  next unless ci_config_path =~ URI.regexp(%w[http https])
+
+                  YAML.dump('include' => [{ 'remote' => ci_config_path }])
+                end
+              end
+
+              def source
+                :remote_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/repository.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class Repository < Source
+              def content
+                strong_memoize(:content) do
+                  next unless file_in_repository?
+
+                  YAML.dump('include' => [{ 'local' => ci_config_path }])
+                end
+              end
+
+              def source
+                :repository_source
+              end
+
+              private
+
+              def file_in_repository?
+                return unless project
+                return unless @pipeline.sha
+
+                project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present?
+              rescue GRPC::NotFound, GRPC::Internal
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/runtime.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class Runtime < Source
+              def content
+                @command.config_content
+              end
+
+              def source
+                # The only case when this source is used is when the config content
+                # is passed in as parameter to Ci::CreatePipelineService.
+                # This would only occur with parent/child pipelines which is being
+                # implemented.
+                # TODO: change source to return :runtime_source
+                # https://gitlab.com/gitlab-org/gitlab/merge_requests/21041
+
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end