Add latest changes from gitlab-org/gitlab@master

app/assets/javascripts/lib/utils/common_utils.js

@@ -177,6 +177,15 @@ export const urlParamsToArray = (path = '') =>
 export const getUrlParamsArray = () => urlParamsToArray(window.location.search);
+/**
+ * Accepts encoding string which includes query params being
+ * sent to URL.
+ *
+ * @param {string} path Query param string
+ *
+ * @returns {object} Query params object containing key-value pairs
+ *                   with both key and values decoded into plain string.
+ */
 export const urlParamsToObject = (path = '') =>
   splitPath(path).reduce((dataParam, filterParam) => {
     if (filterParam === '') {
@@ -185,6 +194,7 @@ export const urlParamsToObject = (path = '') =>
     const data = dataParam;
     let [key, value] = filterParam.split('=');
+    key = /%\w+/g.test(key) ? decodeURIComponent(key) : key;
     const isArray = key.includes('[]');
     key = key.replace('[]', '');
     value = decodeURIComponent(value.replace(/\+/g, ' '));

app/models/clusters/cluster.rb

@@ -24,6 +24,7 @@ module Clusters
     KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN'
     belongs_to :user
+    belongs_to :management_project, class_name: '::Project', optional: true
     has_many :cluster_projects, class_name: 'Clusters::Project'
     has_many :projects, through: :cluster_projects, class_name: '::Project'
@@ -63,6 +64,7 @@ module Clusters
     validate :restrict_modification, on: :update
     validate :no_groups, unless: :group_type?
     validate :no_projects, unless: :project_type?
+    validate :unique_management_project_environment_scope
     after_save :clear_reactive_cache!
@@ -200,6 +202,18 @@ module Clusters
     private
+    def unique_management_project_environment_scope
+      return unless management_project
+
+      duplicate_management_clusters = management_project.management_clusters
+        .where(environment_scope: environment_scope)
+        .where.not(id: id)
+
+      if duplicate_management_clusters.any?
+        errors.add(:environment_scope, "cannot add duplicated environment scope")
+      end
+    end
+
     def instance_domain
       @instance_domain ||= Gitlab::CurrentSettings.auto_devops_domain
     end

app/models/clusters/clusters_hierarchy.rb

@@ -4,8 +4,9 @@ module Clusters
   class ClustersHierarchy
     DEPTH_COLUMN = :depth
-    def initialize(clusterable)
+    def initialize(clusterable, include_management_project: true)
       @clusterable = clusterable
+      @include_management_project = include_management_project
     end
     # Returns clusters in order from deepest to highest group
@@ -24,7 +25,7 @@ module Clusters
     private
-    attr_reader :clusterable
+    attr_reader :clusterable, :include_management_project
     def recursive_cte
       cte = Gitlab::SQL::RecursiveCTE.new(:clusters_cte)
@@ -38,12 +39,25 @@ module Clusters
                      raise ArgumentError, "unknown type for #{clusterable}"
                    end
+      if clusterable.is_a?(::Project) && include_management_project
+        cte << management_clusters_query
+      end
+
       cte << base_query
       cte << parent_query(cte)
       cte
     end
+    # Management clusters should be first in the hierarchy so we use 0 for the
+    # depth column.
+    #
+    # group_parent_id is un-used but we still need to match the same number of
+    # columns as other queries in the CTE.
+    def management_clusters_query
+      clusterable.management_clusters.select([clusters_star, 'NULL AS group_parent_id', "0 AS #{DEPTH_COLUMN}"])
+    end
+
     def group_clusters_base_query
       group_parent_id_alias = alias_as_column(groups[:parent_id], 'group_parent_id')
       join_sources = ::Group.left_joins(:clusters).arel.join_sources

app/models/clusters/platforms/kubernetes.rb

@@ -73,7 +73,7 @@ module Clusters
               .append(key: 'KUBE_CA_PEM_FILE', value: ca_pem, file: true)
           end
-          if !cluster.managed?
+          if !cluster.managed? || cluster.management_project == project
             namespace = Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: project).from_environment_name(environment_name)
             variables

app/models/concerns/deployment_platform.rb

@@ -11,6 +11,10 @@ module DeploymentPlatform
   private
+  def cluster_management_project_enabled?
+    Feature.enabled?(:cluster_management_project, default_enabled: true)
+  end
+
   def find_deployment_platform(environment)
     find_platform_kubernetes_with_cte(environment) ||
       find_instance_cluster_platform_kubernetes(environment: environment)
@@ -18,7 +22,7 @@ module DeploymentPlatform
   # EE would override this and utilize environment argument
   def find_platform_kubernetes_with_cte(_environment)
-    Clusters::ClustersHierarchy.new(self).base_and_ancestors
+    Clusters::ClustersHierarchy.new(self, include_management_project: cluster_management_project_enabled?).base_and_ancestors
       .enabled.default_environment
       .first&.platform_kubernetes
   end

app/models/project.rb

@@ -68,7 +68,7 @@ class Project < ApplicationRecord
     :snippets_access_level, :builds_access_level, :repository_access_level,
     to: :project_feature, allow_nil: true
-  delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage
+  delegate :base_dir, :disk_path, to: :storage
   delegate :scheduled?, :started?, :in_progress?,
     :failed?, :finished?,
@@ -122,8 +122,6 @@ class Project < ApplicationRecord
   # Storage specific hooks
   after_initialize :use_hashed_storage
   after_create :check_repository_absence!
-  after_create :ensure_storage_path_exists
-  after_save :ensure_storage_path_exists, if: :saved_change_to_namespace_id?
   acts_as_ordered_taggable
@@ -247,6 +245,7 @@ class Project < ApplicationRecord
   has_one :cluster_project, class_name: 'Clusters::Project'
   has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster'
   has_many :kubernetes_namespaces, class_name: 'Clusters::KubernetesNamespace'
+  has_many :management_clusters, class_name: 'Clusters::Cluster', foreign_key: :management_project_id, inverse_of: :management_project
   has_many :prometheus_metrics

app/models/storage/hashed_project.rb

@@ -27,14 +27,6 @@ module Storage
       "#{base_dir}/#{disk_hash}" if disk_hash
     end
-    # TODO: remove this method entirely after 12.4 https://gitlab.com/gitlab-org/gitlab/issues/33244
-    # we no longer need ensure_storage_path_exists to call add_namespace since both creating and moving
-    # repositories will be preceded by a mkdir -p in gitaly to ensure the parent of the destination directory
-    # exists.
-    def ensure_storage_path_exists
-      true
-    end
-
     def rename_repo(old_full_path: nil, new_full_path: nil)
       true
     end

app/models/storage/legacy_project.rb

@@ -23,16 +23,6 @@ module Storage
       project.full_path
     end
-    # TODO: remove this method entirely after 12.4 https://gitlab.com/gitlab-org/gitlab/issues/33244
-    # we no longer need ensure_storage_path_exists to call add_namespace since both creating and moving
-    # repositories will be preceded by a mkdir -p in gitaly to ensure the parent of the destination directory
-    # exists.
-    def ensure_storage_path_exists
-      return unless namespace
-
-      true
-    end
-
     def rename_repo(old_full_path: nil, new_full_path: nil)
       old_full_path ||= project.full_path_before_last_save
       new_full_path ||= project.build_full_path

app/services/projects/hashed_storage/migrate_repository_service.rb

@@ -8,7 +8,6 @@ module Projects
         @old_storage_version = project.storage_version
         project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
-        project.ensure_storage_path_exists
         @new_disk_path = project.disk_path

app/services/projects/hashed_storage/rollback_repository_service.rb

@@ -8,7 +8,6 @@ module Projects
         @old_storage_version = project.storage_version
         project.storage_version = nil
-        project.ensure_storage_path_exists
         @new_disk_path = project.disk_path

app/views/search/_form.html.haml

@@ -18,4 +18,3 @@
       = render 'filter'
     .d-flex-center.flex-column.flex-lg-row
       = button_tag _("Search"), class: "btn btn-success btn-search form-control mt-lg-0 ml-lg-1 align-self-end"
-      = render_if_exists 'search/form_elasticsearch'

app/views/search/show.html.haml

@@ -2,9 +2,10 @@
 - page_title @search_term
 - @hide_breadcrumbs = true
-.page-title-holder.d-flex.align-items-center
+.page-title-holder.d-sm-flex.align-items-sm-center
   %h1.page-title<
     = _('Search')
+  = render_if_exists 'search/form_elasticsearch', attrs: { class: 'ml-sm-auto' }
 .prepend-top-default
   = render 'search/form'

changelogs/unreleased/cluster_management_projects.yml

+---
+title: Adds management project for a cluster
+merge_request: 17866
+author:
+type: changed

db/migrate/20190930063627_add_management_project_id_to_clusters.rb

+# frozen_string_literal: true
+
+class AddManagementProjectIdToClusters < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    add_column :clusters, :management_project_id, :integer
+  end
+end

db/migrate/20191001040549_add_management_project_id_index_fk_to_clusters.rb

+# frozen_string_literal: true
+
+class AddManagementProjectIdIndexFkToClusters < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :clusters, :projects, column: :management_project_id, on_delete: :nullify
+    add_concurrent_index :clusters, :management_project_id, where: 'management_project_id IS NOT NULL'
+  end
+
+  def down
+    remove_concurrent_index :clusters, :management_project_id
+    remove_foreign_key_if_exists :clusters, column: :management_project_id
+  end
+end

db/schema.rb

@@ -991,7 +991,9 @@ ActiveRecord::Schema.define(version: 2019_10_04_134055) do
     t.string "domain"
     t.boolean "managed", default: true, null: false
     t.boolean "namespace_per_environment", default: true, null: false
+    t.integer "management_project_id"
     t.index ["enabled"], name: "index_clusters_on_enabled"
+    t.index ["management_project_id"], name: "index_clusters_on_management_project_id", where: "(management_project_id IS NOT NULL)"
     t.index ["user_id"], name: "index_clusters_on_user_id"
   end
@@ -3983,6 +3985,7 @@ ActiveRecord::Schema.define(version: 2019_10_04_134055) do
   add_foreign_key "cluster_projects", "clusters", on_delete: :cascade
   add_foreign_key "cluster_projects", "projects", on_delete: :cascade
   add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade
+  add_foreign_key "clusters", "projects", column: "management_project_id", name: "fk_f05c5e5a42", on_delete: :nullify
   add_foreign_key "clusters", "users", on_delete: :nullify
   add_foreign_key "clusters_applications_cert_managers", "clusters", on_delete: :cascade
   add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade

doc/user/clusters/management_project.md

+# Cluster management project (alpha)
+
+CAUTION: **Warning:**
+This is an _alpha_ feature, and it is subject to change at any time without
+prior notice.
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/17866) in GitLab 12.4
+
+A project can be designated as the management project for a cluster.
+A management project can be used to run deployment jobs with
+Kubernetes
+[`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)
+privileges.
+
+This can be useful for:
+
+- Creating pipelines to install cluster-wide applications into your cluster.
+- Any jobs that require `cluster-admin` privileges.
+
+## Permissions
+
+Only the management project will receive `cluster-admin` privileges. All
+other projects will continue to receive [namespace scoped `edit` level privileges](../project/clusters/index.md#rbac-cluster-resources).
+
+## Usage
+
+### Selecting a cluster management project
+
+This will be implemented as part of [this
+issue](https://gitlab.com/gitlab-org/gitlab/issues/32810).
+
+### Configuring your pipeline
+
+After designating a project as the management project for the cluster,
+write a [`.gitlab-ci,yml`](../../ci/yaml/README.md) in that project. For example:
+
+```yaml
+configure cluster:
+  stage: deploy
+  script: kubectl get namespaces
+  environment:
+    name: production
+```
+
+### Setting the environment scope **(PREMIUM)**
+
+[Environment
+scopes](../project/clusters/index.md#setting-the-environment-scope-premium)
+are usable when associating multiple clusters to the same management
+project.
+
+Each scope can only be used by a single cluster for a management project.
+
+For example, let's say the following Kubernetes clusters are associated
+to a management project:
+
+| Cluster     | Environment scope |
+| ----------- | ----------------- |
+| Development | `*`               |
+| Staging     | `staging`         |
+| Production  | `production`      |
+
+The the following environments set in
+[`.gitlab-ci.yml`](../../ci/yaml/README.md) will deploy to the
+Development, Staging, and Production cluster respectively.
+
+```yaml
+stages:
+- deploy
+
+configure development cluster:
+  stage: deploy
+  script: kubectl get namespaces
+  environment:
+    name: development
+
+configure staging cluster:
+  stage: deploy
+  script: kubectl get namespaces
+  environment:
+    name: staging
+
+configure production cluster:
+  stage: deploy
+  script: kubectl get namespaces
+  environment:
+    name: production
+```
+
+## Disabling this feature
+
+This feature is enabled by default. To disable this feature, disable the
+feature flag `:cluster_management_project`.
+
+To check if the feature flag is enabled on your GitLab instance,
+please ask an administrator to execute the following in a Rails console:
+
+```ruby
+Feature.enabled?(:cluster_management_project)     # Check if it's enabled or not.
+Feature.disable(:cluster_management_project)      # Disable the feature flag.
+```

doc/user/discussions/index.md

@@ -25,7 +25,7 @@ creates another standard comment. Replying to a threaded comment creates a reply
 [Markdown] and [quick actions], just as if you replied from the web.
 NOTE: **Note:**
-There is a limit of 5,000 comments for every object.
+There is a limit of 5,000 comments for every object, for example: issue, epic, and merge request.
 ## Resolvable comments and threads

lib/backup/repository.rb

@@ -80,7 +80,6 @@ module Backup
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{project.full_path} ... "
         path_to_project_bundle = path_to_bundle(project)
-        project.ensure_storage_path_exists
         restore_repo_success = nil
         if File.exist?(path_to_project_bundle)

spec/factories/clusters/clusters.rb

@@ -30,6 +30,10 @@ FactoryBot.define do
       end
     end
+    trait :management_project do
+      management_project factory: :project
+    end
+
     trait :namespace_per_environment_disabled do
       namespace_per_environment { false }
     end