Implement support for CI variables of type file

Add env_var and file as supported types for CI variables. Variables of
type file expose to users existing gitlab-runner behaviour - save
variable value into a temp file and set the path to this file in an ENV
var named after the variable key.

Resolves https://gitlab.com/gitlab-org/gitlab-ce/issues/46806.

app/assets/javascripts/ci_variable_list/ci_variable_list.js

@@ -26,6 +26,10 @@ export default class VariableList {
         selector: '.js-ci-variable-input-id',
         default: '',
       },
+      variable_type: {
+        selector: '.js-ci-variable-input-variable-type',
+        default: 'env_var',
+      },
       key: {
         selector: '.js-ci-variable-input-key',
         default: '',

app/assets/javascripts/ci_variable_list/native_form_variable_list.js

@@ -19,6 +19,7 @@ export default function setupNativeFormVariableList({ container, formField = 'va
     const isTouched = variableList.checkIfRowTouched($lastRow);
     if (!isTouched) {
       $lastRow.find('input, textarea').attr('name', '');
+      $lastRow.find('select').attr('name', '');
     }
   });
 }

app/controllers/groups/variables_controller.rb

@@ -41,7 +41,7 @@ module Groups
     end
     def variable_params_attributes
-      %i[id key secret_value protected masked _destroy]
+      %i[id variable_type key secret_value protected masked _destroy]
     end
     def authorize_admin_build!

app/controllers/projects/pipeline_schedules_controller.rb

@@ -98,7 +98,7 @@ class Projects::PipelineSchedulesController < Projects::ApplicationController
   def schedule_params
     params.require(:schedule)
       .permit(:description, :cron, :cron_timezone, :ref, :active,
-        variables_attributes: [:id, :key, :secret_value, :_destroy] )
+        variables_attributes: [:id, :variable_type, :key, :secret_value, :_destroy] )
   end
   def authorize_play_pipeline_schedule!

app/controllers/projects/pipelines_controller.rb

@@ -169,7 +169,7 @@ class Projects::PipelinesController < Projects::ApplicationController
   end
   def create_params
-    params.require(:pipeline).permit(:ref, variables_attributes: %i[key secret_value])
+    params.require(:pipeline).permit(:ref, variables_attributes: %i[key variable_type secret_value])
   end
   # rubocop: disable CodeReuse/ActiveRecord

app/controllers/projects/variables_controller.rb

@@ -38,6 +38,6 @@ class Projects::VariablesController < Projects::ApplicationController
   end
   def variable_params_attributes
-    %i[id key secret_value protected masked _destroy]
+    %i[id variable_type key secret_value protected masked _destroy]
   end
 end

app/helpers/ci_variables_helper.rb

@@ -20,4 +20,11 @@ module CiVariablesHelper
       true
     end
   end
+
+  def ci_variable_type_options
+    [
+      %w(Variable env_var),
+      %w(File file)
+    ]
+  end
 end

app/models/concerns/has_variable.rb

@@ -4,6 +4,11 @@ module HasVariable
   extend ActiveSupport::Concern
   included do
+    enum variable_type: {
+      env_var: 1,
+      file: 2
+    }
+
     validates :key,
       presence: true,
       length: { maximum: 255 },
@@ -24,6 +29,6 @@ module HasVariable
   end
   def to_runner_variable
-    { key: key, value: value, public: false }
+    { key: key, value: value, public: false, file: file? }
   end
 end

app/views/ci/variables/_variable_row.html.haml

@@ -3,6 +3,7 @@
 - only_key_value = local_assigns.fetch(:only_key_value, false)
 - id = variable&.id
+- variable_type = variable&.variable_type
 - key = variable&.key
 - value = variable&.value
 - is_protected_default = ci_variable_protected_by_default?
@@ -12,6 +13,7 @@
 - id_input_name = "#{form_field}[variables_attributes][][id]"
 - destroy_input_name = "#{form_field}[variables_attributes][][_destroy]"
+- variable_type_input_name = "#{form_field}[variables_attributes][][variable_type]"
 - key_input_name = "#{form_field}[variables_attributes][][key]"
 - value_input_name = "#{form_field}[variables_attributes][][secret_value]"
 - protected_input_name = "#{form_field}[variables_attributes][][protected]"
@@ -21,6 +23,8 @@
   .ci-variable-row-body
     %input.js-ci-variable-input-id{ type: "hidden", name: id_input_name, value: id }
     %input.js-ci-variable-input-destroy{ type: "hidden", name: destroy_input_name }
+    %select.js-ci-variable-input-variable-type.ci-variable-body-item.form-control.select-control{ name: variable_type_input_name }
+      = options_for_select(ci_variable_type_options, variable_type)
     %input.js-ci-variable-input-key.ci-variable-body-item.qa-ci-variable-input-key.form-control{ type: "text",
       name: key_input_name,
       value: key,

changelogs/unreleased/46806-typed-ci-variables.yml

+---
+title: CI variables of type file
+merge_request: 27112
+author:
+type: added

db/migrate/20190415030217_add_variable_type_to_ci_variables.rb

+# frozen_string_literal: true
+
+class AddVariableTypeToCiVariables < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  ENV_VAR_VARIABLE_TYPE = 1
+
+  def up
+    add_column_with_default(:ci_variables, :variable_type, :smallint, default: ENV_VAR_VARIABLE_TYPE)
+  end
+
+  def down
+    remove_column(:ci_variables, :variable_type)
+  end
+end

db/migrate/20190416213556_add_variable_type_to_ci_group_variables.rb

+# frozen_string_literal: true
+
+class AddVariableTypeToCiGroupVariables < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  ENV_VAR_VARIABLE_TYPE = 1
+
+  def up
+    add_column_with_default(:ci_group_variables, :variable_type, :smallint, default: ENV_VAR_VARIABLE_TYPE)
+  end
+
+  def down
+    remove_column(:ci_group_variables, :variable_type)
+  end
+end

db/migrate/20190416213615_add_variable_type_to_ci_pipeline_variables.rb

+# frozen_string_literal: true
+
+class AddVariableTypeToCiPipelineVariables < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  ENV_VAR_VARIABLE_TYPE = 1
+
+  def up
+    add_column_with_default(:ci_pipeline_variables, :variable_type, :smallint, default: ENV_VAR_VARIABLE_TYPE)
+  end
+
+  def down
+    remove_column(:ci_pipeline_variables, :variable_type)
+  end
+end

db/migrate/20190416213631_add_variable_type_to_ci_pipeline_schedule_variables.rb

+# frozen_string_literal: true
+
+class AddVariableTypeToCiPipelineScheduleVariables < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  ENV_VAR_VARIABLE_TYPE = 1
+
+  def up
+    add_column_with_default(:ci_pipeline_schedule_variables, :variable_type, :smallint, default: ENV_VAR_VARIABLE_TYPE)
+  end
+
+  def down
+    remove_column(:ci_pipeline_schedule_variables, :variable_type)
+  end
+end

db/schema.rb

@@ -419,6 +419,7 @@ ActiveRecord::Schema.define(version: 20190426180107) do
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
     t.boolean "masked", default: false, null: false
+    t.integer "variable_type", limit: 2, default: 1, null: false
     t.index ["group_id", "key"], name: "index_ci_group_variables_on_group_id_and_key", unique: true, using: :btree
   end
@@ -458,6 +459,7 @@ ActiveRecord::Schema.define(version: 20190426180107) do
     t.integer "pipeline_schedule_id", null: false
     t.datetime_with_timezone "created_at"
     t.datetime_with_timezone "updated_at"
+    t.integer "variable_type", limit: 2, default: 1, null: false
     t.index ["pipeline_schedule_id", "key"], name: "index_ci_pipeline_schedule_variables_on_schedule_id_and_key", unique: true, using: :btree
   end
@@ -484,6 +486,7 @@ ActiveRecord::Schema.define(version: 20190426180107) do
     t.string "encrypted_value_salt"
     t.string "encrypted_value_iv"
     t.integer "pipeline_id", null: false
+    t.integer "variable_type", limit: 2, default: 1, null: false
     t.index ["pipeline_id", "key"], name: "index_ci_pipeline_variables_on_pipeline_id_and_key", unique: true, using: :btree
   end
@@ -618,6 +621,7 @@ ActiveRecord::Schema.define(version: 20190426180107) do
     t.boolean "protected", default: false, null: false
     t.string "environment_scope", default: "*", null: false
     t.boolean "masked", default: false, null: false
+    t.integer "variable_type", limit: 2, default: 1, null: false
     t.index ["project_id", "key", "environment_scope"], name: "index_ci_variables_on_project_id_and_key_and_environment_scope", unique: true, using: :btree
   end

doc/api/group_level_variables.md

@@ -22,10 +22,12 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a
 [
     {
         "key": "TEST_VARIABLE_1",
+        "variable_type": "env_var",
         "value": "TEST_1"
     },
     {
         "key": "TEST_VARIABLE_2",
+        "variable_type": "env_var",
         "value": "TEST_2"
     }
 ]
@@ -51,6 +53,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a
 ```json
 {
     "key": "TEST_VARIABLE_1",
+    "variable_type": "env_var",
     "value": "TEST_1"
 }
 ```
@@ -63,12 +66,13 @@ Create a new variable.
 POST /groups/:id/variables
 ```
-| Attribute   | Type    | required | Description           |
-|-------------|---------|----------|-----------------------|
-| `id`        | integer/string | yes      | The ID of a group or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user   |
-| `key`       | string  | yes      | The `key` of a variable; must have no more than 255 characters; only `A-Z`, `a-z`, `0-9`, and `_` are allowed |
-| `value`     | string  | yes      | The `value` of a variable |
-| `protected` | boolean | no       | Whether the variable is protected |
+| Attribute       | Type    | required | Description           |
+|-----------------|---------|----------|-----------------------|
+| `id`            | integer/string | yes      | The ID of a group or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user   |
+| `key`           | string  | yes      | The `key` of a variable; must have no more than 255 characters; only `A-Z`, `a-z`, `0-9`, and `_` are allowed |
+| `value`         | string  | yes      | The `value` of a variable |
+| `variable_type` | string  | no       | The type of a variable. Available types are: `env_var` (default) and `file` |
+| `protected`     | boolean | no       | Whether the variable is protected |
 ```
 curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/variables" --form "key=NEW_VARIABLE" --form "value=new value"
@@ -78,6 +82,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitla
 {
     "key": "NEW_VARIABLE",
     "value": "new value",
+    "variable_type": "env_var",
     "protected": false
 }
 ```
@@ -90,12 +95,13 @@ Update a group's variable.
 PUT /groups/:id/variables/:key
 ```
-| Attribute   | Type    | required | Description             |
-|-------------|---------|----------|-------------------------|
-| `id`        | integer/string | yes      | The ID of a group or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user     |
-| `key`       | string  | yes      | The `key` of a variable   |
-| `value`     | string  | yes      | The `value` of a variable |
-| `protected` | boolean | no       | Whether the variable is protected |
+| Attribute       | Type    | required | Description             |
+|-----------------|---------|----------|-------------------------|
+| `id`            | integer/string | yes      | The ID of a group or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user     |
+| `key`           | string  | yes      | The `key` of a variable   |
+| `value`         | string  | yes      | The `value` of a variable |
+| `variable_type` | string  | no       | The type of a variable. Available types are: `env_var` (default) and `file` |
+| `protected`     | boolean | no       | Whether the variable is protected |
 ```
 curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/variables/NEW_VARIABLE" --form "value=updated value"
@@ -105,6 +111,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
 {
     "key": "NEW_VARIABLE",
     "value": "updated value",
+    "variable_type": "env_var",
     "protected": true
 }
 ```

doc/api/pipeline_schedules.md

@@ -88,6 +88,7 @@ curl --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gitlab.example.com/
     "variables": [
         {
             "key": "TEST_VARIABLE_1",
+            "variable_type": "env_var",
             "value": "TEST_1"
         }
     ]
@@ -296,6 +297,7 @@ POST /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables
 | `pipeline_schedule_id` | integer        | yes      | The pipeline schedule id |
 | `key`                  | string         | yes      | The `key` of a variable; must have no more than 255 characters; only `A-Z`, `a-z`, `0-9`, and `_` are allowed |
 | `value`                | string         | yes      | The `value` of a variable |
+| `variable_type`        | string         | no       | The type of a variable. Available types are: `env_var` (default) and `file` |
 ```sh
 curl --request POST --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" --form "key=NEW_VARIABLE" --form "value=new value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables"
@@ -304,6 +306,7 @@ curl --request POST --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" --form "key=N
 ```json
 {
     "key": "NEW_VARIABLE",
+    "variable_type": "env_var",
     "value": "new value"
 }
 ```
@@ -322,6 +325,7 @@ PUT /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables/:key
 | `pipeline_schedule_id` | integer        | yes      | The pipeline schedule id |
 | `key`                  | string         | yes      | The `key` of a variable   |
 | `value`                | string         | yes      | The `value` of a variable |
+| `variable_type`        | string         | no       | The type of a variable. Available types are: `env_var` (default) and `file` |
 ```sh
 curl --request PUT --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" --form "value=updated value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables/NEW_VARIABLE"
@@ -331,6 +335,7 @@ curl --request PUT --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" --form "value=
 {
     "key": "NEW_VARIABLE",
     "value": "updated value"
+    "variable_type": "env_var",
 }
 ```

doc/api/pipelines.md

@@ -114,6 +114,7 @@ Example of response
 [
   {
     "key": "RUN_NIGHTLY_BUILD",
+    "variable_type": "env_var",
     "value": "true"
   },
   {
@@ -135,7 +136,7 @@ POST /projects/:id/pipeline
 |------------|---------|----------|---------------------|
 | `id`       | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
 | `ref`       | string | yes      | Reference to commit |
-| `variables` | array | no | An array containing the variables available in the pipeline, matching the structure [{ 'key' => 'UPLOAD_TO_S3', 'value' => 'true' }] |
+| `variables` | array | no | An array containing the variables available in the pipeline, matching the structure [{ 'key' => 'UPLOAD_TO_S3', 'variable_type' => 'file', 'value' => 'true' }] |
 ```
 curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/pipeline?ref=master"

doc/api/project_level_variables.md

@@ -20,10 +20,12 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a
 [
     {
         "key": "TEST_VARIABLE_1",
+        "variable_type": "env_var",
         "value": "TEST_1"
     },
     {
         "key": "TEST_VARIABLE_2",
+        "variable_type": "env_var",
         "value": "TEST_2"
     }
 ]
@@ -49,6 +51,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a
 ```json
 {
     "key": "TEST_VARIABLE_1",
+    "variable_type": "env_var",
     "value": "TEST_1"
 }
 ```
@@ -61,12 +64,13 @@ Create a new variable.
 POST /projects/:id/variables
 ```
-| Attribute   | Type    | required | Description           |
-|-------------|---------|----------|-----------------------|
-| `id`        | integer/string | yes      | The ID of a project or [urlencoded NAMESPACE/PROJECT_NAME of the project](README.md#namespaced-path-encoding) owned by the authenticated user   |
-| `key`       | string  | yes      | The `key` of a variable; must have no more than 255 characters; only `A-Z`, `a-z`, `0-9`, and `_` are allowed |
-| `value`     | string  | yes      | The `value` of a variable |
-| `protected` | boolean | no       | Whether the variable is protected |
+| Attribute       | Type    | required | Description           |
+|-----------------|---------|----------|-----------------------|
+| `id`            | integer/string | yes      | The ID of a project or [urlencoded NAMESPACE/PROJECT_NAME of the project](README.md#namespaced-path-encoding) owned by the authenticated user   |
+| `key`           | string  | yes      | The `key` of a variable; must have no more than 255 characters; only `A-Z`, `a-z`, `0-9`, and `_` are allowed |
+| `value`         | string  | yes      | The `value` of a variable |
+| `variable_type` | string  | no       | The type of a variable. Available types are: `env_var` (default) and `file` |
+| `protected`     | boolean | no       | Whether the variable is protected |
 ```
 curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/variables" --form "key=NEW_VARIABLE" --form "value=new value"
@@ -76,6 +80,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitla
 {
     "key": "NEW_VARIABLE",
     "value": "new value",
+    "variable_type": "env_var",
     "protected": false
 }
 ```
@@ -88,12 +93,13 @@ Update a project's variable.
 PUT /projects/:id/variables/:key
 ```
-| Attribute   | Type    | required | Description             |
-|-------------|---------|----------|-------------------------|
-| `id`        | integer/string | yes      | The ID of a project or [urlencoded NAMESPACE/PROJECT_NAME of the project](README.md#namespaced-path-encoding) owned by the authenticated user     |
-| `key`       | string  | yes      | The `key` of a variable   |
-| `value`     | string  | yes      | The `value` of a variable |
-| `protected` | boolean | no       | Whether the variable is protected |
+| Attribute       | Type    | required | Description             |
+|-----------------|---------|----------|-------------------------|
+| `id`            | integer/string | yes      | The ID of a project or [urlencoded NAMESPACE/PROJECT_NAME of the project](README.md#namespaced-path-encoding) owned by the authenticated user     |
+| `key`           | string  | yes      | The `key` of a variable   |
+| `value`         | string  | yes      | The `value` of a variable |
+| `variable_type` | string  | no       | The type of a variable. Available types are: `env_var` (default) and `file` |
+| `protected`     | boolean | no       | Whether the variable is protected |
 ```
 curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/variables/NEW_VARIABLE" --form "value=updated value"
@@ -103,6 +109,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
 {
     "key": "NEW_VARIABLE",
     "value": "updated value",
+    "variable_type": "env_var",
     "protected": true
 }
 ```

lib/api/entities.rb

@@ -1288,7 +1288,7 @@ module API
     end
     class Variable < Grape::Entity
-      expose :key, :value
+      expose :variable_type, :key, :value
       expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) }
     end