Bring scoped environment variables to core

As decided in https://gitlab.com/gitlab-org/gitlab-ce/issues/53593

app/controllers/projects/variables_controller.rb

@@ -38,6 +38,6 @@ class Projects::VariablesController < Projects::ApplicationController
   end
   def variable_params_attributes
-    %i[id variable_type key secret_value protected masked _destroy]
+    %i[id variable_type key secret_value protected masked environment_scope _destroy]
   end
 end

app/models/ci/variable.rb

@@ -6,6 +6,7 @@ module Ci
     include HasVariable
     include Presentable
     include Maskable
+    prepend HasEnvironmentScope
     belongs_to :project

app/models/concerns/has_environment_scope.rb

+# frozen_string_literal: true
+
+module HasEnvironmentScope
+  extend ActiveSupport::Concern
+
+  prepended do
+    validates(
+      :environment_scope,
+      presence: true,
+      format: { with: ::Gitlab::Regex.environment_scope_regex,
+                message: ::Gitlab::Regex.environment_scope_regex_message }
+    )
+
+    ##
+    # Select rows which have a scope that matches the given environment name.
+    # Rows are ordered by relevance, by default. The most relevant row is
+    # placed at the end of a list.
+    #
+    # options:
+    #   - relevant_only: (boolean)
+    #     You can get the most relevant row only. Other rows are not be
+    #     selected even if its scope matches the environment name.
+    #     This is equivalent to using `#last` from SQL standpoint.
+    #
+    scope :on_environment, -> (environment_name, relevant_only: false) do
+      order_direction = relevant_only ? 'DESC' : 'ASC'
+
+      where = <<~SQL
+        environment_scope IN (:wildcard, :environment_name) OR
+          :environment_name LIKE
+            #{::Gitlab::SQL::Glob.to_like('environment_scope')}
+      SQL
+
+      order = <<~SQL
+        CASE environment_scope
+          WHEN :wildcard THEN 0
+          WHEN :environment_name THEN 2
+          ELSE 1
+        END #{order_direction}
+      SQL
+
+      values = {
+        wildcard: '*',
+        environment_name: environment_name
+      }
+
+      sanitized_order_sql = sanitize_sql_array([order, values])
+
+      # The query is trying to find variables with scopes matching the
+      # current environment name. Suppose the environment name is
+      # 'review/app', and we have variables with environment scopes like:
+      # * variable A: review
+      # * variable B: review/app
+      # * variable C: review/*
+      # * variable D: *
+      # And the query should find variable B, C, and D, because it would
+      # try to convert the scope into a LIKE pattern for each variable:
+      # * A: review
+      # * B: review/app
+      # * C: review/%
+      # * D: %
+      # Note that we'll match % and _ literally therefore we'll escape them.
+      # In this case, B, C, and D would match. We also want to prioritize
+      # the exact matched name, and put * last, and everything else in the
+      # middle. So the order should be: D < C < B
+      relation = where(where, values)
+        .order(Arel.sql(sanitized_order_sql)) # `order` cannot escape for us!
+
+      relation = relation.limit(1) if relevant_only
+
+      relation
+    end
+  end
+
+  def environment_scope=(new_environment_scope)
+    super(new_environment_scope.to_s.strip)
+  end
+end

app/models/project.rb

@@ -1828,11 +1828,16 @@ class Project < ApplicationRecord
   end
   def ci_variables_for(ref:, environment: nil)
-    # EE would use the environment
-    if protected_for?(ref)
-      variables
+    result = if protected_for?(ref)
+               variables
+             else
+               variables.unprotected
+             end
+
+    if environment
+      result.on_environment(environment)
     else
-      variables.unprotected
+      result.where(environment_scope: '*')
     end
   end

app/serializers/variable_entity.rb

@@ -7,4 +7,5 @@ class VariableEntity < Grape::Entity
   expose :protected?, as: :protected
   expose :masked?, as: :masked
+  expose :environment_scope
 end

app/views/ci/variables/_environment_scope.html.haml

+- form_field = local_assigns.fetch(:form_field, nil)
+- variable = local_assigns.fetch(:variable, nil)
+
+- if @project
+  - environment_scope = variable&.environment_scope || '*'
+  - environment_scope_label = environment_scope == '*' ? s_('CiVariable|All environments') : environment_scope
+
+  %input{ type: "hidden", name: "#{form_field}[variables_attributes][][environment_scope]", value: environment_scope }
+  = dropdown_tag(environment_scope_label,
+                 options: { wrapper_class: 'ci-variable-body-item js-variable-environment-dropdown-wrapper',
+                            toggle_class: 'js-variable-environment-toggle wide',
+                            filter: true,
+                            dropdown_class: "dropdown-menu-selectable",
+                            placeholder: s_('CiVariable|Search environments'),
+                            footer_content: true,
+                            data: { selected: environment_scope } }) do
+    %ul.dropdown-footer-list
+      %li
+        %button{ class: "dropdown-create-new-item-button js-dropdown-create-new-item", title: s_('CiVariable|New environment') }
+          = s_('CiVariable|Create wildcard')
+          %code

app/views/ci/variables/_environment_scope_header.html.haml

+.bold.table-section.section-15.append-right-10
+  = s_('CiVariables|Scope')

changelogs/unreleased/bring-scoped-environment-variables-to-core.yml

+---
+title: Bring scoped environment variables to core
+merge_request: 30779
+author:
+type: changed

doc/README.md

@@ -279,7 +279,7 @@ The following documentation relates to the DevOps **Release** stage:
 | [Canary Deployments](user/project/canary_deployments.md) **(PREMIUM)**                                                              | Employ a popular CI strategy where a small portion of the fleet is updated to the new version first.                                  |
 | [Deploy Boards](user/project/deploy_boards.md) **(PREMIUM)**                                                                        | View the current health and status of each CI environment running on Kubernetes, displaying the status of the pods in the deployment. |
 | [Environments and deployments](ci/environments.md)                                                                                  | With environments, you can control the continuous deployment of your software within GitLab.                                          |
-| [Environment-specific variables](ci/variables/README.md#limiting-environment-scopes-of-environment-variables-premium) **(PREMIUM)** | Limit scope of variables to specific environments.                                                                                    |
+| [Environment-specific variables](ci/variables/README.md#limiting-environment-scopes-of-environment-variables) | Limit scope of variables to specific environments.                                                                                    |
 | [GitLab CI/CD](ci/README.md)                                                                                                        | Explore the features and capabilities of Continuous Deployment and Delivery with GitLab.                                              |
 | [GitLab Pages](user/project/pages/index.md)                                                                                         | Build, test, and deploy a static site directly from GitLab.                                                                           |
 | [Protected Runners](ci/runners/README.md#protected-runners)                                                                         | Select Runners to only pick jobs for protected branches and tags.                                                                     |

doc/api/project_level_variables.md

@@ -74,7 +74,7 @@ POST /projects/:id/variables
 | `variable_type`     | string  | no       | The type of a variable. Available types are: `env_var` (default) and `file` |
 | `protected`         | boolean | no       | Whether the variable is protected |
 | `masked`            | boolean | no       | Whether the variable is masked |
-| `environment_scope` | string  | no       | The `environment_scope` of the variable **(PREMIUM)** |
+| `environment_scope` | string  | no       | The `environment_scope` of the variable |
 ```
 curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/variables" --form "key=NEW_VARIABLE" --form "value=new value"
@@ -108,7 +108,7 @@ PUT /projects/:id/variables/:key
 | `variable_type`     | string  | no       | The type of a variable. Available types are: `env_var` (default) and `file` |
 | `protected`         | boolean | no       | Whether the variable is protected |
 | `masked`            | boolean | no       | Whether the variable is masked |
-| `environment_scope` | string  | no       | The `environment_scope` of the variable **(PREMIUM)** |
+| `environment_scope` | string  | no       | The `environment_scope` of the variable |
 ```
 curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/variables/NEW_VARIABLE" --form "value=updated value"

doc/ci/environments.md

@@ -692,7 +692,7 @@ with `review/` would have that particular variable.
 Some GitLab features can behave differently for each environment.
 For example, you can
-[create a secret variable to be injected only into a production environment](variables/README.md#limiting-environment-scopes-of-environment-variables-premium). **(PREMIUM)**
+[create a secret variable to be injected only into a production environment](variables/README.md#limiting-environment-scopes-of-environment-variables).
 In most cases, these features use the _environment specs_ mechanism, which offers
 an efficient way to implement scoping within each environment group.

doc/ci/variables/README.md

@@ -393,7 +393,7 @@ Protected variables can be added by going to your project's
 Once you set them, they will be available for all subsequent pipelines.
-### Limiting environment scopes of environment variables **(PREMIUM)**
+### Limiting environment scopes of environment variables
 You can limit the environment scope of a variable by
 [defining which environments][envs] it can be available for.

doc/topics/autodevops/index.md

@@ -190,7 +190,7 @@ Those environments are tied to jobs that use [Auto Deploy](#auto-deploy), so
 except for the environment scope, they would also need to have a different
 domain they would be deployed to. This is why you need to define a separate
 `KUBE_INGRESS_BASE_DOMAIN` variable for all the above
-[based on the environment](../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables-premium).
+[based on the environment](../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables).
 The following table is an example of how the three different clusters would
 be configured.
@@ -662,10 +662,10 @@ repo or by specifying a project variable:
 You can also make use of the `HELM_UPGRADE_EXTRA_ARGS` environment variable to override the default values in the `values.yaml` file in the [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app).
 To apply your own `values.yaml` file to all Helm upgrade commands in Auto Deploy set `HELM_UPGRADE_EXTRA_ARGS` to `--values my-values.yaml`.
-### Custom Helm chart per environment **(PREMIUM)**
+### Custom Helm chart per environment
 You can specify the use of a custom Helm chart per environment by scoping the environment variable
-to the desired environment. See [Limiting environment scopes of variables](../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables-premium).
+to the desired environment. See [Limiting environment scopes of variables](../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables).
 ### Customizing `.gitlab-ci.yml`

doc/user/group/clusters/index.md

@@ -86,7 +86,7 @@ The domain should have a wildcard DNS configured to the Ingress IP address.
 When adding more than one Kubernetes cluster to your project, you need to differentiate
 them with an environment scope. The environment scope associates clusters with
 [environments](../../../ci/environments.md) similar to how the
-[environment-specific variables](../../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables-premium)
+[environment-specific variables](../../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables)
 work.
 While evaluating which environment matches the environment scope of a

doc/user/project/clusters/index.md

@@ -468,7 +468,7 @@ If you don't want to use GitLab Runner in privileged mode, either:
 When adding more than one Kubernetes cluster to your project, you need to differentiate
 them with an environment scope. The environment scope associates clusters with [environments](../../../ci/environments.md) similar to how the
-[environment-specific variables](../../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables-premium) work.
+[environment-specific variables](../../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables) work.
 The default environment scope is `*`, which means all jobs, regardless of their
 environment, will use that cluster. Each scope can only be used by a single

lib/api/entities.rb

@@ -1346,6 +1346,7 @@ module API
       expose :variable_type, :key, :value
       expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) }
       expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) }
+      expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) }
     end
     class Pipeline < PipelineBasic

lib/api/helpers/variables_helpers.rb

-# frozen_string_literal: true
-
-module API
-  module Helpers
-    module VariablesHelpers
-      extend ActiveSupport::Concern
-      extend Grape::API::Helpers
-
-      params :optional_params_ee do
-      end
-    end
-  end
-end

lib/api/variables.rb

@@ -7,8 +7,6 @@ module API
     before { authenticate! }
     before { authorize! :admin_build, user_project }
-    helpers Helpers::VariablesHelpers
-
     helpers do
       def filter_variable_parameters(params)
         # This method exists so that EE can more easily filter out certain
@@ -59,8 +57,7 @@ module API
         optional :protected, type: Boolean, desc: 'Whether the variable is protected'
         optional :masked, type: Boolean, desc: 'Whether the variable is masked'
         optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var'
-
-        use :optional_params_ee
+        optional :environment_scope, type: String, desc: 'The environment_scope of the variable'
       end
       post ':id/variables' do
         variable_params = declared_params(include_missing: false)
@@ -84,8 +81,7 @@ module API
         optional :protected, type: Boolean, desc: 'Whether the variable is protected'
         optional :masked, type: Boolean, desc: 'Whether the variable is masked'
         optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file'
-
-        use :optional_params_ee
+        optional :environment_scope, type: String, desc: 'The environment_scope of the variable'
       end
       # rubocop: disable CodeReuse/ActiveRecord
       put ':id/variables/:key' do

lib/gitlab/regex.rb

@@ -46,6 +46,18 @@ module Gitlab
       "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'"
     end
+    def environment_scope_regex_chars
+      "#{environment_name_regex_chars}\\*"
+    end
+
+    def environment_scope_regex
+      @environment_scope_regex ||= /\A[#{environment_scope_regex_chars}]+\z/.freeze
+    end
+
+    def environment_scope_regex_message
+      "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', '*' and spaces"
+    end
+
     def kubernetes_namespace_regex
       /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/
     end

locale/gitlab.pot

@@ -2263,6 +2263,9 @@ msgstr ""
 msgid "CiVariables|Remove variable row"
 msgstr ""
+msgid "CiVariables|Scope"
+msgstr ""
+
 msgid "CiVariables|Specify variable values to be used in this run. The values specified in %{linkStart}CI/CD settings%{linkEnd} will be used as default"
 msgstr ""
@@ -2284,15 +2287,24 @@ msgstr ""
 msgid "CiVariable|All environments"
 msgstr ""
+msgid "CiVariable|Create wildcard"
+msgstr ""
+
 msgid "CiVariable|Error occurred while saving variables"
 msgstr ""
 msgid "CiVariable|Masked"
 msgstr ""
+msgid "CiVariable|New environment"
+msgstr ""
+
 msgid "CiVariable|Protected"
 msgstr ""
+msgid "CiVariable|Search environments"
+msgstr ""
+
 msgid "CiVariable|Toggle masked"
 msgstr ""