Merge branch '32831-single-deploy-of-runner-in-k8s-cluster' into 'master'

Extend Cluster Applications to install GitLab Runner to Kubernetes cluster

Closes #32831

See merge request gitlab-org/gitlab-ce!17134

app/assets/javascripts/clusters/components/applications.vue

@@ -99,12 +99,6 @@
           </p>
         `;
       },
-      gitlabRunnerDescription() {
-        return _.escape(s__(
-          `ClusterIntegration|GitLab Runner is the open source project that is used to run your jobs
-          and send the results back to GitLab.`,
-        ));
-      },
       prometheusDescription() {
         return sprintf(
           _.escape(s__(
@@ -256,6 +250,22 @@
           >
           </div>
         </application-row>
+        <application-row
+          id="runner"
+          :title="applications.runner.title"
+          title-link="https://docs.gitlab.com/runner/"
+          :status="applications.runner.status"
+          :status-reason="applications.runner.statusReason"
+          :request-status="applications.runner.requestStatus"
+          :request-reason="applications.runner.requestReason"
+        >
+          <div slot="description">
+            {{ s__(`ClusterIntegration|GitLab Runner connects to this
+              project's repository and executes CI/CD jobs,
+              pushing results back and deploying,
+              applications to production.`) }}
+          </div>
+        </application-row>
         <!--
           NOTE: Don't forget to update `clusters.scss`
           min-height for this block and uncomment `application_spec` tests

app/models/clusters/applications/helm.rb

@@ -15,7 +15,7 @@ module Clusters
       end
       def install_command
-        Gitlab::Kubernetes::Helm::InstallCommand.new(name, install_helm: true)
+        Gitlab::Kubernetes::Helm::InitCommand.new(name)
       end
     end
   end

app/models/clusters/applications/ingress.rb

@@ -5,6 +5,7 @@ module Clusters
       include ::Clusters::Concerns::ApplicationCore
       include ::Clusters::Concerns::ApplicationStatus
+      include ::Clusters::Concerns::ApplicationData
       include AfterCommitQueue
       default_value_for :ingress_type, :nginx
@@ -29,12 +30,12 @@ module Clusters
         'stable/nginx-ingress'
       end
-      def chart_values_file
-        "#{Rails.root}/vendor/#{name}/values.yaml"
-      end
-
       def install_command
-        Gitlab::Kubernetes::Helm::InstallCommand.new(name, chart: chart, chart_values_file: chart_values_file)
+        Gitlab::Kubernetes::Helm::InstallCommand.new(
+          name,
+          chart: chart,
+          values: values
+        )
       end
       def schedule_status_update

app/models/clusters/applications/prometheus.rb

@@ -7,6 +7,7 @@ module Clusters
       include ::Clusters::Concerns::ApplicationCore
       include ::Clusters::Concerns::ApplicationStatus
+      include ::Clusters::Concerns::ApplicationData
       default_value_for :version, VERSION
@@ -30,12 +31,12 @@ module Clusters
         80
       end
-      def chart_values_file
-        "#{Rails.root}/vendor/#{name}/values.yaml"
-      end
-
       def install_command
-        Gitlab::Kubernetes::Helm::InstallCommand.new(name, chart: chart, chart_values_file: chart_values_file)
+        Gitlab::Kubernetes::Helm::InstallCommand.new(
+          name,
+          chart: chart,
+          values: values
+        )
       end
       def proxy_client

app/models/clusters/applications/runner.rb

+module Clusters
+  module Applications
+    class Runner < ActiveRecord::Base
+      VERSION = '0.1.13'.freeze
+
+      self.table_name = 'clusters_applications_runners'
+
+      include ::Clusters::Concerns::ApplicationCore
+      include ::Clusters::Concerns::ApplicationStatus
+      include ::Clusters::Concerns::ApplicationData
+
+      belongs_to :runner, class_name: 'Ci::Runner', foreign_key: :runner_id
+      delegate :project, to: :cluster
+
+      default_value_for :version, VERSION
+
+      def chart
+        "#{name}/gitlab-runner"
+      end
+
+      def repository
+        'https://charts.gitlab.io'
+      end
+
+      def values
+        content_values.to_yaml
+      end
+
+      def install_command
+        Gitlab::Kubernetes::Helm::InstallCommand.new(
+          name,
+          chart: chart,
+          values: values,
+          repository: repository
+        )
+      end
+
+      private
+
+      def ensure_runner
+        runner || create_and_assign_runner
+      end
+
+      def create_and_assign_runner
+        transaction do
+          project.runners.create!(name: 'kubernetes-cluster', tag_list: %w(kubernetes cluster)).tap do |runner|
+            update!(runner_id: runner.id)
+          end
+        end
+      end
+
+      def gitlab_url
+        Gitlab::Routing.url_helpers.root_url(only_path: false)
+      end
+
+      def specification
+        {
+          "gitlabUrl" => gitlab_url,
+          "runnerToken" => ensure_runner.token
+        }
+      end
+
+      def content_values
+        specification.merge(YAML.load_file(chart_values_file))
+      end
+    end
+  end
+end

app/models/clusters/cluster.rb

@@ -7,7 +7,8 @@ module Clusters
     APPLICATIONS = {
       Applications::Helm.application_name => Applications::Helm,
       Applications::Ingress.application_name => Applications::Ingress,
-      Applications::Prometheus.application_name => Applications::Prometheus
+      Applications::Prometheus.application_name => Applications::Prometheus,
+      Applications::Runner.application_name => Applications::Runner
     }.freeze
     belongs_to :user
@@ -23,6 +24,7 @@ module Clusters
     has_one :application_helm, class_name: 'Clusters::Applications::Helm'
     has_one :application_ingress, class_name: 'Clusters::Applications::Ingress'
     has_one :application_prometheus, class_name: 'Clusters::Applications::Prometheus'
+    has_one :application_runner, class_name: 'Clusters::Applications::Runner'
     accepts_nested_attributes_for :provider_gcp, update_only: true
     accepts_nested_attributes_for :platform_kubernetes, update_only: true
@@ -68,7 +70,8 @@ module Clusters
       [
         application_helm || build_application_helm,
         application_ingress || build_application_ingress,
-        application_prometheus || build_application_prometheus
+        application_prometheus || build_application_prometheus,
+        application_runner || build_application_runner
       ]
     end

app/models/clusters/concerns/application_data.rb

+module Clusters
+  module Concerns
+    module ApplicationData
+      extend ActiveSupport::Concern
+
+      included do
+        def repository
+          nil
+        end
+
+        def values
+          File.read(chart_values_file)
+        end
+
+        private
+
+        def chart_values_file
+          "#{Rails.root}/vendor/#{name}/values.yaml"
+        end
+      end
+    end
+  end
+end

app/views/projects/clusters/show.html.haml

@@ -10,6 +10,7 @@
   install_helm_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :helm),
   install_ingress_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :ingress),
   install_prometheus_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :prometheus),
+  install_runner_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :runner),
   toggle_status: @cluster.enabled? ? 'true': 'false',
   cluster_status: @cluster.status_name,
   cluster_status_reason: @cluster.status_reason,

changelogs/unreleased/32831-single-deploy-of-runner-in-k8s-cluster.yml

+---
+title: Allow installation of GitLab Runner with a single click
+merge_request: 17134
+author:
+type: added

db/migrate/20180214155405_create_clusters_applications_runners.rb

+class CreateClustersApplicationsRunners < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :clusters_applications_runners do |t|
+      t.references :cluster, null: false, foreign_key: { on_delete: :cascade }
+      t.references :runner, references: :ci_runners
+      t.index :runner_id
+      t.index :cluster_id, unique: true
+      t.integer :status, null: false
+      t.timestamps_with_timezone null: false
+      t.string :version, null: false
+      t.text :status_reason
+    end
+
+    add_concurrent_foreign_key :clusters_applications_runners, :ci_runners,
+      column: :runner_id,
+      on_delete: :nullify
+  end
+
+  def down
+    if foreign_keys_for(:clusters_applications_runners, :runner_id).any?
+      remove_foreign_key :clusters_applications_runners, column: :runner_id
+    end
+
+    drop_table :clusters_applications_runners
+  end
+end

db/schema.rb

@@ -582,6 +582,19 @@ ActiveRecord::Schema.define(version: 20180301084653) do
     t.datetime_with_timezone "updated_at", null: false
   end
+  create_table "clusters_applications_runners", force: :cascade do |t|
+    t.integer "cluster_id", null: false
+    t.integer "runner_id"
+    t.integer "status", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.string "version", null: false
+    t.text "status_reason"
+  end
+
+  add_index "clusters_applications_runners", ["cluster_id"], name: "index_clusters_applications_runners_on_cluster_id", unique: true, using: :btree
+  add_index "clusters_applications_runners", ["runner_id"], name: "index_clusters_applications_runners_on_runner_id", using: :btree
+
   create_table "container_repositories", force: :cascade do |t|
     t.integer "project_id", null: false
     t.string "name", null: false
@@ -1988,6 +2001,8 @@ ActiveRecord::Schema.define(version: 20180301084653) do
   add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade
   add_foreign_key "clusters", "users", on_delete: :nullify
   add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade
+  add_foreign_key "clusters_applications_runners", "ci_runners", column: "runner_id", name: "fk_02de2ded36", on_delete: :nullify
+  add_foreign_key "clusters_applications_runners", "clusters", on_delete: :cascade
   add_foreign_key "container_repositories", "projects"
   add_foreign_key "deploy_keys_projects", "projects", name: "fk_58a901ca7e", on_delete: :cascade
   add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade

doc/user/project/clusters/index.md

@@ -120,6 +120,7 @@ added directly to your configured cluster. Those applications are needed for
 | [Helm Tiller](https://docs.helm.sh/) | 10.2+ | Helm is a package manager for Kubernetes and is required to install all the other applications. It will be automatically installed as a dependency when you try to install a different app. It is installed in its own pod inside the cluster which can run the `helm` CLI in a safe environment. |
 | [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) | 10.2+ | Ingress can provide load balancing, SSL termination, and name-based virtual hosting. It acts as a web proxy for your applications and is useful if you want to use [Auto DevOps](../../../topics/autodevops/index.md) or deploy your own web apps. |
 | [Prometheus](https://prometheus.io/docs/introduction/overview/) | 10.4+ | Prometheus is an open-source monitoring and alerting system useful to supervise your deployed applications |
+| [GitLab Runner](https://docs.gitlab.com/runner/) | 10.6+ | GitLab Runner is the open source project that is used to run your jobs and send the results back to GitLab. It is used in conjunction with [GitLab CI](https://about.gitlab.com/features/gitlab-ci-cd/), the open-source continuous integration service included with GitLab that coordinates the jobs. |
 ## Getting the external IP address

lib/gitlab/kubernetes/config_map.rb

+module Gitlab
+  module Kubernetes
+    class ConfigMap
+      def initialize(name, values)
+        @name = name
+        @values = values
+      end
+
+      def generate
+        resource = ::Kubeclient::Resource.new
+        resource.metadata = metadata
+        resource.data = { values: values }
+        resource
+      end
+
+      private
+
+      attr_reader :name, :values
+
+      def metadata
+        {
+          name: config_map_name,
+          namespace: namespace,
+          labels: { name: config_map_name }
+        }
+      end
+
+      def config_map_name
+        "values-content-configuration-#{name}"
+      end
+
+      def namespace
+        Gitlab::Kubernetes::Helm::NAMESPACE
+      end
+    end
+  end
+end

lib/gitlab/kubernetes/helm/api.rb

@@ -9,7 +9,8 @@ module Gitlab
         def install(command)
           @namespace.ensure_exists!
-          @kubeclient.create_pod(pod_resource(command))
+          create_config_map(command) if command.config_map?
+          @kubeclient.create_pod(command.pod_resource)
         end
         ##
@@ -33,8 +34,10 @@ module Gitlab
         private
-        def pod_resource(command)
-          Gitlab::Kubernetes::Helm::Pod.new(command, @namespace.name, @kubeclient).generate
+        def create_config_map(command)
+          command.config_map_resource.tap do |config_map_resource|
+            @kubeclient.create_config_map(config_map_resource)
+          end
         end
       end
     end

lib/gitlab/kubernetes/helm/base_command.rb

+module Gitlab
+  module Kubernetes
+    module Helm
+      class BaseCommand
+        attr_reader :name
+
+        def initialize(name)
+          @name = name
+        end
+
+        def pod_resource
+          Gitlab::Kubernetes::Helm::Pod.new(self, namespace).generate
+        end
+
+        def generate_script
+          <<~HEREDOC
+            set -eo pipefail
+            apk add -U ca-certificates openssl >/dev/null
+            wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null
+            mv /tmp/linux-amd64/helm /usr/bin/
+          HEREDOC
+        end
+
+        def config_map?
+          false
+        end
+
+        def pod_name
+          "install-#{name}"
+        end
+
+        private
+
+        def namespace
+          Gitlab::Kubernetes::Helm::NAMESPACE
+        end
+      end
+    end
+  end
+end

lib/gitlab/kubernetes/helm/init_command.rb

+module Gitlab
+  module Kubernetes
+    module Helm
+      class InitCommand < BaseCommand
+        def generate_script
+          super + [
+            init_helm_command
+          ].join("\n")
+        end
+
+        private
+
+        def init_helm_command
+          "helm init >/dev/null"
+        end
+      end
+    end
+  end
+end

lib/gitlab/kubernetes/helm/install_command.rb

 module Gitlab
   module Kubernetes
     module Helm
-      class InstallCommand
-        attr_reader :name, :install_helm, :chart, :chart_values_file
-        def initialize(name, install_helm: false, chart: false, chart_values_file: false)
+      class InstallCommand < BaseCommand
+        attr_reader :name, :chart, :repository, :values
+        def initialize(name, chart:, values:, repository: nil)
           @name = name
-          @install_helm = install_helm
           @chart = chart
-          @chart_values_file = chart_values_file
+          @values = values
+          @repository = repository
         end
-        def pod_name
-          "install-#{name}"
+        def generate_script
+          super + [
+            init_command,
+            repository_command,
+            script_command
+          ].compact.join("\n")
         end
-        def generate_script(namespace_name)
-          [
-            install_dps_command,
-            init_command,
-            complete_command(namespace_name)
-          ].join("\n")
+        def config_map?
+          true
+        end
+
+        def config_map_resource
+          Gitlab::Kubernetes::ConfigMap.new(name, values).generate
         end
         private
         def init_command
-          if install_helm
-            'helm init >/dev/null'
-          else
-            'helm init --client-only >/dev/null'
-          end
+          'helm init --client-only >/dev/null'
         end
-        def complete_command(namespace_name)
-          return unless chart
-
-          if chart_values_file
-            "helm install #{chart} --name #{name} --namespace #{namespace_name} -f /data/helm/#{name}/config/values.yaml >/dev/null"
-          else
-            "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null"
-          end
+        def repository_command
+          "helm repo add #{name} #{repository}" if repository
         end
-        def install_dps_command
+        def script_command
           <<~HEREDOC
-            set -eo pipefail
-            apk add -U ca-certificates openssl >/dev/null
-            wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null
-            mv /tmp/linux-amd64/helm /usr/bin/
+          helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null
           HEREDOC
         end
       end

lib/gitlab/kubernetes/helm/pod.rb

@@ -2,18 +2,17 @@ module Gitlab
   module Kubernetes
     module Helm
       class Pod
-        def initialize(command, namespace_name, kubeclient)
+        def initialize(command, namespace_name)
           @command = command
           @namespace_name = namespace_name
-          @kubeclient = kubeclient
         end
         def generate
           spec = { containers: [container_specification], restartPolicy: 'Never' }
-          if command.chart_values_file
-            create_config_map
+          if command.config_map?
             spec[:volumes] = volumes_specification
+            spec[:containers][0][:volumeMounts] = volume_mounts_specification
           end
           ::Kubeclient::Resource.new(metadata: metadata, spec: spec)
@@ -21,18 +20,16 @@ module Gitlab
         private
-        attr_reader :command, :namespace_name, :kubeclient
+        attr_reader :command, :namespace_name, :kubeclient, :config_map
         def container_specification
-          container = {
+          {
             name: 'helm',
             image: 'alpine:3.6',
             env: generate_pod_env(command),
             command: %w(/bin/sh),
             args: %w(-c $(COMMAND_SCRIPT))
           }
-          container[:volumeMounts] = volume_mounts_specification if command.chart_values_file
-          container
         end
         def labels
@@ -50,13 +47,12 @@ module Gitlab
           }
         end
-        def volume_mounts_specification
-          [
-            {
-              name: 'configuration-volume',
-              mountPath: "/data/helm/#{command.name}/config"
-            }
-          ]
+        def generate_pod_env(command)
+          {
+            HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION,
+            TILLER_NAMESPACE: namespace_name,
+            COMMAND_SCRIPT: command.generate_script
+          }.map { |key, value| { name: key, value: value } }
         end
         def volumes_specification
@@ -71,23 +67,13 @@ module Gitlab
           ]
         end
-        def generate_pod_env(command)
-          {
-            HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION,
-            TILLER_NAMESPACE: namespace_name,
-            COMMAND_SCRIPT: command.generate_script(namespace_name)
-          }.map { |key, value| { name: key, value: value } }
-        end
-
-        def create_config_map
-          resource = ::Kubeclient::Resource.new
-          resource.metadata = {
-            name: "values-content-configuration-#{command.name}",
-            namespace: namespace_name,
-            labels: { name: "values-content-configuration-#{command.name}" }
-          }
-          resource.data = { values: File.read(command.chart_values_file) }
-          kubeclient.create_config_map(resource)
+        def volume_mounts_specification
+          [
+            {
+              name: 'configuration-volume',
+              mountPath: "/data/helm/#{command.name}/config"
+            }
+          ]
         end
       end
     end

spec/factories/clusters/applications/helm.rb

@@ -34,5 +34,6 @@ FactoryBot.define do
     factory :clusters_applications_ingress, class: Clusters::Applications::Ingress
     factory :clusters_applications_prometheus, class: Clusters::Applications::Prometheus
+    factory :clusters_applications_runner, class: Clusters::Applications::Runner
   end
 end

spec/javascripts/clusters/components/applications_spec.js

@@ -38,11 +38,9 @@ describe('Applications', () => {
       expect(vm.$el.querySelector('.js-cluster-application-row-prometheus')).toBeDefined();
     });
-    /* * /
     it('renders a row for GitLab Runner', () => {
       expect(vm.$el.querySelector('.js-cluster-application-row-runner')).toBeDefined();
     });
-    /* */
   });
   describe('Ingress application', () => {