Add latest changes from gitlab-org/gitlab@master

.gitlab/issue_templates/Feature proposal.md

@@ -15,6 +15,8 @@
 * [Sidney (Systems Administrator)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sidney-systems-administrator)
 * [Sam (Security Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sam-security-analyst)
 * [Dana (Data Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#dana-data-analyst)
+* [Simone (Software Engineer in Test)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#simone-software-engineer-in-test)
+* [Allison (Application Ops)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#allison-application-ops)
 Personas are described at https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/ -->

app/models/clusters/cluster.rb

@@ -249,9 +249,13 @@ module Clusters
       platform_kubernetes.kubeclient if kubernetes?
     end
-    def kubernetes_namespace_for(environment)
+    def kubernetes_namespace_for(environment, deployable: environment.last_deployable)
+      if deployable && environment.project_id != deployable.project_id
+        raise ArgumentError, 'environment.project_id must match deployable.project_id'
+      end
+
       managed_namespace(environment) ||
-        ci_configured_namespace(environment) ||
+        ci_configured_namespace(deployable) ||
         default_namespace(environment)
     end
@@ -318,8 +322,11 @@ module Clusters
       ).execute&.namespace
     end
-    def ci_configured_namespace(environment)
-      environment.last_deployable&.expanded_kubernetes_namespace
+    def ci_configured_namespace(deployable)
+      # YAML configuration of namespaces not supported for managed clusters
+      return if managed?
+
+      deployable&.expanded_kubernetes_namespace
     end
     def default_namespace(environment)

changelogs/unreleased/dblessing-add-scim-identities.yml

+---
+title: Create scim_identities table in preparation for newer SCIM features in the
+  future
+merge_request: 26124
+author:
+type: added

changelogs/unreleased/fix-deployment-namespace-resolution.yml

+---
+title: Fix Kubernetes namespace resolution for new DeploymentCluster records
+merge_request: 25853
+author:
+type: fixed

db/migrate/20200227164113_create_scim_identities.rb

+# frozen_string_literal: true
+
+class CreateScimIdentities < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    create_table :scim_identities do |t|
+      t.references :group, foreign_key: { to_table: :namespaces, on_delete: :cascade }, null: false
+      t.references :user, index: false, foreign_key: { on_delete: :cascade }, null: false
+      t.timestamps_with_timezone
+      t.boolean :active, default: false
+      t.string :extern_uid, null: false, limit: 255
+
+      t.index 'LOWER(extern_uid),group_id', name: 'index_scim_identities_on_lower_extern_uid_and_group_id', unique: true
+      t.index [:user_id, :group_id], unique: true
+    end
+  end
+end

db/schema.rb

@@ -3819,6 +3819,18 @@ ActiveRecord::Schema.define(version: 2020_03_04_160823) do
     t.index ["group_id"], name: "index_saml_providers_on_group_id"
   end
+  create_table "scim_identities", force: :cascade do |t|
+    t.bigint "group_id", null: false
+    t.bigint "user_id", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.boolean "active", default: false
+    t.string "extern_uid", limit: 255, null: false
+    t.index "lower((extern_uid)::text), group_id", name: "index_scim_identities_on_lower_extern_uid_and_group_id", unique: true
+    t.index ["group_id"], name: "index_scim_identities_on_group_id"
+    t.index ["user_id", "group_id"], name: "index_scim_identities_on_user_id_and_group_id", unique: true
+  end
+
   create_table "scim_oauth_access_tokens", id: :serial, force: :cascade do |t|
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
@@ -5037,6 +5049,8 @@ ActiveRecord::Schema.define(version: 2020_03_04_160823) do
   add_foreign_key "reviews", "projects", on_delete: :cascade
   add_foreign_key "reviews", "users", column: "author_id", on_delete: :nullify
   add_foreign_key "saml_providers", "namespaces", column: "group_id", on_delete: :cascade
+  add_foreign_key "scim_identities", "namespaces", column: "group_id", on_delete: :cascade
+  add_foreign_key "scim_identities", "users", on_delete: :cascade
   add_foreign_key "scim_oauth_access_tokens", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "security_scans", "ci_builds", column: "build_id", on_delete: :cascade
   add_foreign_key "self_managed_prometheus_alert_events", "environments", on_delete: :cascade

doc/api/project_clusters.md

@@ -177,16 +177,16 @@ Parameters:
 | Attribute | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
 | `id` | integer | yes | The ID of the project owned by the authenticated user |
-| `name` | String | yes | The name of the cluster |
-| `domain` | String | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster |
+| `name` | string | yes | The name of the cluster |
+| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster |
 | `enabled` | boolean | no | Determines if cluster is active or not, defaults to true |
 | `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true |
-| `platform_kubernetes_attributes[api_url]` | String | yes | The URL to access the Kubernetes API |
-| `platform_kubernetes_attributes[token]` | String | yes | The token to authenticate against Kubernetes |
-| `platform_kubernetes_attributes[ca_cert]` | String | no | TLS certificate (needed if API is using a self-signed TLS certificate |
-| `platform_kubernetes_attributes[namespace]` | String | no | The unique namespace related to the project |
-| `platform_kubernetes_attributes[authorization_type]` | String | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. |
-| `environment_scope` | String | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** |
+| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API |
+| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes |
+| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. |
+| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project |
+| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. |
+| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** |
 Example request:
@@ -271,14 +271,14 @@ Parameters:
 | --------- | ---- | -------- | ----------- |
 | `id` | integer | yes | The ID of the project owned by the authenticated user |
 | `cluster_id` | integer | yes | The ID of the cluster |
-| `name` | String | no | The name of the cluster |
-| `domain` | String | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster |
+| `name` | string | no | The name of the cluster |
+| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster |
 | `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster |
-| `platform_kubernetes_attributes[api_url]` | String | no | The URL to access the Kubernetes API |
-| `platform_kubernetes_attributes[token]` | String | no | The token to authenticate against Kubernetes |
-| `platform_kubernetes_attributes[ca_cert]` | String | no | TLS certificate (needed if API is using a self-signed TLS certificate |
-| `platform_kubernetes_attributes[namespace]` | String | no | The unique namespace related to the project |
-| `environment_scope` | String | no | The associated environment to the cluster **(PREMIUM)** |
+| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API |
+| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes |
+| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. |
+| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project |
+| `environment_scope` | string | no | The associated environment to the cluster **(PREMIUM)** |
 NOTE: **Note:**
 `name`, `api_url`, `ca_cert` and `token` can only be updated if the cluster was added

doc/ci/variables/predefined_variables.md

@@ -33,7 +33,7 @@ future GitLab releases.**
 | `CI_COMMIT_DESCRIPTION`                       | 10.8   | all    | The description of the commit: the message without first line, if the title is shorter than 100 characters; full message in other case.                                                                                                                                                                                                                    |
 | `CI_COMMIT_MESSAGE`                           | 10.8   | all    | The full commit message.                                                                                                                                                                                                                                                                                                                                   |
 | `CI_COMMIT_REF_NAME`                          | 9.0    | all    | The branch or tag name for which project is built                                                                                                                                                                                                                                                                                                          |
-| `CI_COMMIT_REF_PROTECTED`                     | 11.11  | all    | If the job is running on a protected branch                                                                                                                                                                                                                                                                                                                |
+| `CI_COMMIT_REF_PROTECTED`                     | 11.11  | all    | `true` if the job is running on a protected branch, `false` if not                                                                                                                                                                                                                                                                                                               |
 | `CI_COMMIT_REF_SLUG`                          | 9.0    | all    | `$CI_COMMIT_REF_NAME` lowercased, shortened to 63 bytes, and with everything except `0-9` and `a-z` replaced with `-`. No leading / trailing `-`. Use in URLs, host names and domain names.                                                                                                                                                                |
 | `CI_COMMIT_SHA`                               | 9.0    | all    | The commit revision for which project is built                                                                                                                                                                                                                                                                                                             |
 | `CI_COMMIT_SHORT_SHA`                         | 11.7   | all    | The first eight characters of `CI_COMMIT_SHA`                                                                                                                                                                                                                                                                                                              |

doc/integration/vault.md

@@ -87,11 +87,11 @@ The following assumes you already have Vault installed and running.
    1. If the `OIDC` method is not currently selected, open the dropdown and select it.
    1. Click the **Sign in With GitLab** button, which will open a modal window:
-   ![Sign into Vault with GitLab](img/sign_into_vault_with_gitlab_v12_6.png)
+      ![Sign into Vault with GitLab](img/sign_into_vault_with_gitlab_v12_6.png)
    1. Click **Authorize** on the modal to allow Vault to sign in through GitLab. This will redirect you back to your Vault UI as a signed-in user.
-   ![Authorize Vault to connect with GitLab](img/authorize_vault_with_gitlab_v12_6.png)
+      ![Authorize Vault to connect with GitLab](img/authorize_vault_with_gitlab_v12_6.png)
 1. **Sign in using the Vault CLI** (optional):

doc/user/gitlab_com/index.md

@@ -530,6 +530,14 @@ On GitLab.com, projects, groups, and snippets created
 As of GitLab 12.2 (July 2019), projects, groups, and snippets have the
 [**Internal** visibility](../../public_access/public_access.md#internal-projects) setting [disabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/issues/12388).
+### SSH maximum number of connections
+
+GitLab.com defines the maximum number of concurrent, unauthenticated SSH connections by
+using the [MaxStartups setting](http://man.openbsd.org/sshd_config.5#MaxStartups).
+If more than the maximum number of allowed connections occur concurrently, they are
+dropped and users get
+[an `ssh_exchange_identification` error](../../topics/git/troubleshooting_git.md#ssh_exchange_identification-error).
+
 ## GitLab.com Logging
 We use [Fluentd](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#fluentd) to parse our logs. Fluentd sends our logs to

doc/user/project/clusters/serverless/index.md

@@ -384,7 +384,8 @@ The sample function can now be triggered from any HTTP client using a simple `PO
 ### Secrets
-To access your Kubernetes secrets from within your function, the secrets should be created under the namespace of your serverless deployment.
+To access your Kubernetes secrets from within your function, the secrets should be created under the namespace of your serverless deployment and specified in your `serverless.yml` file as above.
+You can create secrets in several ways. The following sections show some examples.
 #### CLI example

lib/gitlab/ci/pipeline/seed/deployment.rb

@@ -23,12 +23,12 @@ module Gitlab
             # non-environment job.
             return unless deployment.valid? && deployment.environment.persisted?
-            if cluster_id = deployment.environment.deployment_platform&.cluster_id
+            if cluster = deployment.environment.deployment_platform&.cluster
               # double write cluster_id until 12.9: https://gitlab.com/gitlab-org/gitlab/issues/202628
-              deployment.cluster_id = cluster_id
+              deployment.cluster_id = cluster.id
               deployment.deployment_cluster = ::DeploymentCluster.new(
-                cluster_id: cluster_id,
-                kubernetes_namespace: deployment.environment.deployment_namespace
+                cluster_id: cluster.id,
+                kubernetes_namespace: cluster.kubernetes_namespace_for(deployment.environment, deployable: job)
               )
             end

spec/factories/deployments.rb

@@ -7,7 +7,7 @@ FactoryBot.define do
     tag { false }
     user { nil }
     project { nil }
-    deployable factory: :ci_build
+    deployable { association :ci_build, environment: environment.name, project: environment.project }
     environment factory: :environment
     after(:build) do |deployment, evaluator|

spec/lib/gitlab/ci/pipeline/seed/deployment_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 describe Gitlab::Ci::Pipeline::Seed::Deployment do
-  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:project, refind: true) { create(:project, :repository) }
   let(:pipeline) do
     create(:ci_pipeline, project: project,
            sha: 'b83d6e391c22777fca1ed3012fce84f633d7fed0')
@@ -25,10 +25,12 @@ describe Gitlab::Ci::Pipeline::Seed::Deployment do
       let(:attributes) do
         {
           environment: 'production',
-          options: { environment: { name: 'production' } }
+          options: { environment: { name: 'production', **kubernetes_options } }
         }
       end
+      let(:kubernetes_options) { {} }
+
       it 'returns a deployment object with environment' do
         expect(subject).to be_a(Deployment)
         expect(subject.iid).to be_present
@@ -38,14 +40,30 @@ describe Gitlab::Ci::Pipeline::Seed::Deployment do
       end
       context 'when environment has deployment platform' do
-        let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) }
+        let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project], managed: managed_cluster) }
+        let(:managed_cluster) { true }
         it 'sets the cluster and deployment_cluster' do
           expect(subject.cluster).to eq(cluster) # until we stop double writing in 12.9: https://gitlab.com/gitlab-org/gitlab/issues/202628
-          expect(subject.deployment_cluster).to have_attributes(
-            cluster_id: cluster.id,
-            kubernetes_namespace: subject.environment.deployment_namespace
-          )
+          expect(subject.deployment_cluster.cluster).to eq(cluster)
+        end
+
+        context 'when a custom namespace is given' do
+          let(:kubernetes_options) { { kubernetes: { namespace: 'the-custom-namespace' } } }
+
+          context 'when cluster is managed' do
+            it 'does not set the custom namespace' do
+              expect(subject.deployment_cluster.kubernetes_namespace).not_to eq('the-custom-namespace')
+            end
+          end
+
+          context 'when cluster is not managed' do
+            let(:managed_cluster) { false }
+
+            it 'sets the custom namespace' do
+              expect(subject.deployment_cluster.kubernetes_namespace).to eq('the-custom-namespace')
+            end
+          end
         end
       end

spec/models/clusters/cluster_spec.rb

@@ -674,59 +674,59 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
   end
   describe '#kubernetes_namespace_for' do
-    let(:cluster) { create(:cluster, :group) }
-    let(:environment) { create(:environment, last_deployable: build) }
-    let(:build) { create(:ci_build) }
-    subject { cluster.kubernetes_namespace_for(environment) }
-    before do
-      expect(Clusters::KubernetesNamespaceFinder).to receive(:new)
-        .with(cluster, project: environment.project, environment_name: environment.name)
-        .and_return(double(execute: persisted_namespace))
-
-      allow(build).to receive(:expanded_kubernetes_namespace)
-        .and_return(ci_configured_namespace)
+    subject { cluster.kubernetes_namespace_for(environment, deployable: build) }
+    let(:environment_name) { 'the-environment-name' }
+    let(:environment) { create(:environment, name: environment_name, project: cluster.project, last_deployable: build) }
+    let(:build) { create(:ci_build, environment: environment_name, project: cluster.project) }
+    let(:cluster) { create(:cluster, :project, managed: managed_cluster) }
+    let(:managed_cluster) { true }
+    let(:default_namespace) { Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: cluster.project).from_environment_slug(environment.slug) }
+    let(:build_options) { {} }
+    it 'validates the project id' do
+      environment.project_id = build.project_id + 1
+      expect { subject }.to raise_error ArgumentError, 'environment.project_id must match deployable.project_id'
     end
-    context 'no persisted namespace exists and namespace is not specified in CI template' do
-      let(:persisted_namespace) { nil }
-      let(:ci_configured_namespace) { nil }
-      let(:namespace_generator) { double }
-      let(:default_namespace) { 'a-default-namespace' }
+    context 'when environment has no last_deployable' do
+      let(:build) { nil }
+      it { is_expected.to eq default_namespace }
+    end
+    context 'when cluster is managed' do
       before do
-        expect(Gitlab::Kubernetes::DefaultNamespace).to receive(:new)
-          .with(cluster, project: environment.project)
-          .and_return(namespace_generator)
-        expect(namespace_generator).to receive(:from_environment_slug)
-          .with(environment.slug)
-          .and_return(default_namespace)
+        build.options = { environment: { kubernetes: { namespace: 'ci yaml namespace' } } }
       end
-      it { is_expected.to eq default_namespace }
-    end
-
-    context 'persisted namespace exists' do
-      let(:persisted_namespace) { create(:cluster_kubernetes_namespace) }
-      let(:ci_configured_namespace) { nil }
-      it { is_expected.to eq persisted_namespace.namespace }
+      it 'returns the cached namespace if present, ignoring CI config' do
+        cached_namespace = create(:cluster_kubernetes_namespace, cluster: cluster, environment: environment, namespace: 'the name', service_account_token: 'some token')
+        expect(subject).to eq cached_namespace.namespace
+      end
+      it 'returns the default namespace when no cached namespace, ignoring CI config' do
+        expect(subject).to eq default_namespace
+      end
     end
-    context 'namespace is specified in CI template' do
-      let(:persisted_namespace) { nil }
-      let(:ci_configured_namespace) { 'ci-configured-namespace' }
-      it { is_expected.to eq ci_configured_namespace }
-    end
-    context 'persisted namespace exists and namespace is also specifed in CI template' do
-      let(:persisted_namespace) { create(:cluster_kubernetes_namespace) }
-      let(:ci_configured_namespace) { 'ci-configured-namespace' }
-      it { is_expected.to eq persisted_namespace.namespace }
+    context 'when cluster is not managed' do
+      let(:managed_cluster) { false }
+      it 'returns the cached namespace if present, regardless of CI config' do
+        cached_namespace = create(:cluster_kubernetes_namespace, cluster: cluster, environment: environment, namespace: 'the name', service_account_token: 'some token')
+        build.options = { environment: { kubernetes: { namespace: 'ci yaml namespace' } } }
+        expect(subject).to eq cached_namespace.namespace
+      end
+      it 'returns the CI YAML namespace when configured' do
+        build.options = { environment: { kubernetes: { namespace: 'ci yaml namespace' } } }
+        expect(subject).to eq 'ci yaml namespace'
+      end
+      it 'returns the default namespace when no namespace is configured' do
+        expect(subject).to eq default_namespace
+      end
     end
   end