Upgrade cluster applications, starting with runner

app/assets/javascripts/clusters/clusters_bundle.js

@@ -6,7 +6,13 @@ import Flash from '../flash';
 import Poll from '../lib/utils/poll';
 import initSettingsPanels from '../settings_panels';
 import eventHub from './event_hub';
-import { APPLICATION_STATUS, REQUEST_SUBMITTED, REQUEST_FAILURE } from './constants';
+import {
+  APPLICATION_STATUS,
+  REQUEST_SUBMITTED,
+  REQUEST_FAILURE,
+  UPGRADE_REQUESTED,
+  UPGRADE_REQUEST_FAILURE,
+} from './constants';
 import ClustersService from './services/clusters_service';
 import ClustersStore from './stores/clusters_store';
 import Applications from './components/applications.vue';
@@ -120,11 +126,17 @@ export default class Clusters {
   addListeners() {
     if (this.showTokenButton) this.showTokenButton.addEventListener('click', this.showToken);
     eventHub.$on('installApplication', this.installApplication);
+    eventHub.$on('upgradeApplication', data => this.upgradeApplication(data));
+    eventHub.$on('upgradeFailed', appId => this.upgradeFailed(appId));
+    eventHub.$on('dismissUpgradeSuccess', appId => this.dismissUpgradeSuccess(appId));
   }
   removeListeners() {
     if (this.showTokenButton) this.showTokenButton.removeEventListener('click', this.showToken);
     eventHub.$off('installApplication', this.installApplication);
+    eventHub.$off('upgradeApplication', this.upgradeApplication);
+    eventHub.$off('upgradeFailed', this.upgradeFailed);
+    eventHub.$off('dismissUpgradeSuccess', this.dismissUpgradeSuccess);
   }
   initPolling() {
@@ -245,6 +257,21 @@ export default class Clusters {
     });
   }
+  upgradeApplication(data) {
+    const appId = data.id;
+    this.store.updateAppProperty(appId, 'requestStatus', UPGRADE_REQUESTED);
+    this.store.updateAppProperty(appId, 'status', APPLICATION_STATUS.UPDATING);
+    this.service.installApplication(appId, data.params).catch(() => this.upgradeFailed(appId));
+  }
+
+  upgradeFailed(appId) {
+    this.store.updateAppProperty(appId, 'requestStatus', UPGRADE_REQUEST_FAILURE);
+  }
+
+  dismissUpgradeSuccess(appId) {
+    this.store.updateAppProperty(appId, 'requestStatus', null);
+  }
+
   destroy() {
     this.destroyed = true;

app/assets/javascripts/clusters/components/application_row.vue

 <script>
 /* eslint-disable vue/require-default-prop */
+import { GlLink } from '@gitlab/ui';
+import TimeagoTooltip from '../../vue_shared/components/time_ago_tooltip.vue';
 import { s__, sprintf } from '../../locale';
 import eventHub from '../event_hub';
 import identicon from '../../vue_shared/components/identicon.vue';
 import loadingButton from '../../vue_shared/components/loading_button.vue';
-import { APPLICATION_STATUS, REQUEST_SUBMITTED, REQUEST_FAILURE } from '../constants';
+import {
+  APPLICATION_STATUS,
+  REQUEST_SUBMITTED,
+  REQUEST_FAILURE,
+  UPGRADE_REQUESTED,
+} from '../constants';
 export default {
   components: {
     loadingButton,
     identicon,
+    TimeagoTooltip,
+    GlLink,
   },
   props: {
     id: {
@@ -54,6 +63,18 @@ export default {
       type: String,
       required: false,
     },
+    version: {
+      type: String,
+      required: false,
+    },
+    chartRepo: {
+      type: String,
+      required: false,
+    },
+    upgradeAvailable: {
+      type: Boolean,
+      required: false,
+    },
     installApplicationRequestParams: {
       type: Object,
       required: false,
@@ -78,7 +99,8 @@ export default {
       return (
         this.status === APPLICATION_STATUS.INSTALLED ||
         this.status === APPLICATION_STATUS.UPDATED ||
-        this.status === APPLICATION_STATUS.UPDATING
+        this.status === APPLICATION_STATUS.UPDATING ||
+        this.status === APPLICATION_STATUS.UPDATE_ERRORED
       );
     },
     canInstall() {
@@ -146,6 +168,69 @@ export default {
         title: this.title,
       });
     },
+    versionLabel() {
+      if (this.upgradeFailed) {
+        return s__('ClusterIntegration|Upgrade failed');
+      } else if (this.isUpgrading) {
+        return s__('ClusterIntegration|Upgrading');
+      }
+
+      return s__('ClusterIntegration|Upgraded');
+    },
+    upgradeRequested() {
+      return this.requestStatus === UPGRADE_REQUESTED;
+    },
+    upgradeSuccessful() {
+      return this.status === APPLICATION_STATUS.UPDATED;
+    },
+    upgradeFailed() {
+      if (this.isUpgrading) {
+        return false;
+      }
+
+      return this.status === APPLICATION_STATUS.UPDATE_ERRORED;
+    },
+    upgradeFailureDescription() {
+      return sprintf(
+        s__(
+          'ClusterIntegration|Something went wrong when upgrading %{title}. Please check the logs and try again.',
+        ),
+        {
+          title: this.title,
+        },
+      );
+    },
+    upgradeSuccessDescription() {
+      return sprintf(s__('ClusterIntegration|%{title} upgraded successfully.'), {
+        title: this.title,
+      });
+    },
+    upgradeButtonLabel() {
+      let label;
+      if (this.upgradeAvailable && !this.upgradeFailed && !this.isUpgrading) {
+        label = s__('ClusterIntegration|Upgrade');
+      } else if (this.isUpgrading) {
+        label = s__('ClusterIntegration|Upgrading');
+      } else if (this.upgradeFailed) {
+        label = s__('ClusterIntegration|Retry upgrade');
+      }
+
+      return label;
+    },
+    isUpgrading() {
+      // Since upgrading is handled asynchronously on the backend we need this check to prevent any delay on the frontend
+      return (
+        this.status === APPLICATION_STATUS.UPDATING ||
+        (this.upgradeRequested && !this.upgradeSuccessful)
+      );
+    },
+  },
+  watch: {
+    status() {
+      if (this.status === APPLICATION_STATUS.UPDATE_ERRORED) {
+        eventHub.$emit('upgradeFailed', this.id);
+      }
+    },
   },
   methods: {
     installClicked() {
@@ -154,6 +239,15 @@ export default {
         params: this.installApplicationRequestParams,
       });
     },
+    upgradeClicked() {
+      eventHub.$emit('upgradeApplication', {
+        id: this.id,
+        params: this.installApplicationRequestParams,
+      });
+    },
+    dismissUpgradeSuccess() {
+      eventHub.$emit('dismissUpgradeSuccess', this.id);
+    },
   },
 };
 </script>
@@ -207,6 +301,51 @@ export default {
             </li>
           </ul>
         </div>
+
+        <div
+          v-if="(upgradeSuccessful || upgradeFailed) && !upgradeAvailable"
+          class="form-text text-muted label p-0 js-cluster-application-upgrade-details"
+        >
+          {{ versionLabel }}
+
+          <span v-if="upgradeSuccessful"> to</span>
+
+          <gl-link
+            v-if="upgradeSuccessful"
+            :href="chartRepo"
+            target="_blank"
+            class="js-cluster-application-upgrade-version"
+          >
+            chart v{{ version }}
+          </gl-link>
+        </div>
+
+        <div
+          v-if="upgradeFailed && !isUpgrading"
+          class="bs-callout bs-callout-danger cluster-application-banner mt-2 mb-0 js-cluster-application-upgrade-failure-message"
+        >
+          {{ upgradeFailureDescription }}
+        </div>
+
+        <div
+          v-if="upgradeRequested && upgradeSuccessful"
+          class="bs-callout bs-callout-success cluster-application-banner mt-2 mb-0 p-0 pl-3"
+        >
+          {{ upgradeSuccessDescription }}
+
+          <button class="close cluster-application-banner-close" @click="dismissUpgradeSuccess">
+            &times;
+          </button>
+        </div>
+
+        <loading-button
+          v-if="upgradeAvailable || upgradeFailed || isUpgrading"
+          class="btn btn-primary js-cluster-application-upgrade-button mt-2"
+          :loading="isUpgrading"
+          :disabled="isUpgrading"
+          :label="upgradeButtonLabel"
+          @click="upgradeClicked"
+        />
       </div>
       <div
         :class="{ 'section-25': showManageButton, 'section-15': !showManageButton }"

app/assets/javascripts/clusters/components/applications.vue

@@ -362,6 +362,9 @@ export default {
         :status-reason="applications.runner.statusReason"
         :request-status="applications.runner.requestStatus"
         :request-reason="applications.runner.requestReason"
+        :version="applications.runner.version"
+        :chart-repo="applications.runner.chartRepo"
+        :upgrade-available="applications.runner.upgradeAvailable"
         :disabled="!helmInstalled"
         title-link="https://docs.gitlab.com/runner/"
       >

app/assets/javascripts/clusters/constants.js

@@ -12,15 +12,19 @@ export const APPLICATION_STATUS = {
   SCHEDULED: 'scheduled',
   INSTALLING: 'installing',
   INSTALLED: 'installed',
-  UPDATED: 'updated',
   UPDATING: 'updating',
+  UPDATED: 'updated',
+  UPDATE_ERRORED: 'update_errored',
   ERROR: 'errored',
 };
 // These are only used client-side
 export const REQUEST_SUBMITTED = 'request-submitted';
 export const REQUEST_FAILURE = 'request-failure';
+export const UPGRADE_REQUESTED = 'upgrade-requested';
+export const UPGRADE_REQUEST_FAILURE = 'upgrade-request-failure';
 export const INGRESS = 'ingress';
 export const JUPYTER = 'jupyter';
 export const KNATIVE = 'knative';
+export const RUNNER = 'runner';
 export const CERT_MANAGER = 'cert_manager';

app/assets/javascripts/clusters/stores/clusters_store.js

 import { s__ } from '../../locale';
 import { parseBoolean } from '../../lib/utils/common_utils';
-import { INGRESS, JUPYTER, KNATIVE, CERT_MANAGER } from '../constants';
+import { INGRESS, JUPYTER, KNATIVE, CERT_MANAGER, RUNNER } from '../constants';
 export default class ClusterStore {
   constructor() {
@@ -40,6 +40,9 @@ export default class ClusterStore {
           statusReason: null,
           requestStatus: null,
           requestReason: null,
+          version: null,
+          chartRepo: 'https://gitlab.com/charts/gitlab-runner',
+          upgradeAvailable: null,
         },
         prometheus: {
           title: s__('ClusterIntegration|Prometheus'),
@@ -100,7 +103,13 @@ export default class ClusterStore {
     this.state.statusReason = serverState.status_reason;
     serverState.applications.forEach(serverAppEntry => {
-      const { name: appId, status, status_reason: statusReason } = serverAppEntry;
+      const {
+        name: appId,
+        status,
+        status_reason: statusReason,
+        version,
+        update_available: upgradeAvailable,
+      } = serverAppEntry;
       this.state.applications[appId] = {
         ...(this.state.applications[appId] || {}),
@@ -124,6 +133,9 @@ export default class ClusterStore {
           serverAppEntry.hostname || this.state.applications.knative.hostname;
         this.state.applications.knative.externalIp =
           serverAppEntry.external_ip || this.state.applications.knative.externalIp;
+      } else if (appId === RUNNER) {
+        this.state.applications.runner.version = version;
+        this.state.applications.runner.upgradeAvailable = upgradeAvailable;
       }
     });
   }

app/assets/stylesheets/pages/clusters.scss

@@ -58,6 +58,20 @@
   }
 }
+.cluster-application-banner {
+  height: 45px;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+}
+
+.cluster-application-banner-close {
+  align-self: flex-start;
+  font-weight: 500;
+  font-size: 20px;
+  margin: $gl-padding-8 14px 0 0;
+}
+
 .cluster-application-description {
   flex: 1;
 }

app/models/clusters/applications/prometheus.rb

@@ -53,11 +53,11 @@ module Clusters
       end
       def upgrade_command(values)
-        ::Gitlab::Kubernetes::Helm::UpgradeCommand.new(
-          name,
+        ::Gitlab::Kubernetes::Helm::InstallCommand.new(
+          name: name,
           version: VERSION,
-          chart: chart,
           rbac: cluster.platform_kubernetes_rbac?,
+          chart: chart,
           files: files_with_replaced_values(values)
         )
       end

app/models/clusters/concerns/application_status.rb

@@ -20,7 +20,7 @@ module Clusters
           state :update_errored, value: 6
           event :make_scheduled do
-            transition [:installable, :errored] => :scheduled
+            transition [:installable, :errored, :installed, :updated, :update_errored] => :scheduled
           end
           event :make_installing do
@@ -29,16 +29,19 @@ module Clusters
           event :make_installed do
             transition [:installing] => :installed
+            transition [:updating] => :updated
           end
           event :make_errored do
-            transition any => :errored
+            transition any - [:updating] => :errored
+            transition [:updating] => :update_errored
           end
           event :make_updating do
-            transition [:installed, :updated, :update_errored] => :updating
+            transition [:installed, :updated, :update_errored, :scheduled] => :updating
           end
+          # Deprecated
           event :make_updated do
             transition [:updating] => :updated
           end
@@ -74,6 +77,10 @@ module Clusters
         end
       end
+      def updateable?
+        installed? || updated? || update_errored?
+      end
+
       def available?
         installed? || updated?
       end

app/models/clusters/concerns/application_version.rb

@@ -12,6 +12,10 @@ module Clusters
           end
         end
       end
+
+      def update_available?
+        version != self.class.const_get(:VERSION)
+      end
     end
   end
 end

app/serializers/cluster_application_entity.rb

@@ -8,4 +8,5 @@ class ClusterApplicationEntity < Grape::Entity
   expose :external_ip, if: -> (e, _) { e.respond_to?(:external_ip) }
   expose :hostname, if: -> (e, _) { e.respond_to?(:hostname) }
   expose :email, if: -> (e, _) { e.respond_to?(:email) }
+  expose :update_available?, as: :update_available, if: -> (e, _) { e.respond_to?(:update_available?) }
 end

app/services/clusters/applications/check_installation_progress_service.rb

@@ -4,7 +4,7 @@ module Clusters
   module Applications
     class CheckInstallationProgressService < BaseHelmService
       def execute
-        return unless app.installing?
+        return unless operation_in_progress?
         case installation_phase
         when Gitlab::Kubernetes::Pod::SUCCEEDED
@@ -16,11 +16,16 @@ module Clusters
         end
       rescue Kubeclient::HttpError => e
         log_error(e)
-        app.make_errored!("Kubernetes error: #{e.error_code}") unless app.errored?
+
+        app.make_errored!("Kubernetes error: #{e.error_code}")
       end
       private
+      def operation_in_progress?
+        app.installing? || app.updating?
+      end
+
       def on_success
         app.make_installed!
       ensure
@@ -28,13 +33,13 @@ module Clusters
       end
       def on_failed
-        app.make_errored!("Installation failed. Check pod logs for #{install_command.pod_name} for more details.")
+        app.make_errored!("Operation failed. Check pod logs for #{pod_name} for more details.")
       end
       def check_timeout
         if timeouted?
           begin
-            app.make_errored!("Installation timed out. Check pod logs for #{install_command.pod_name} for more details.")
+            app.make_errored!("Operation timed out. Check pod logs for #{pod_name} for more details.")
           end
         else
           ClusterWaitForAppInstallationWorker.perform_in(
@@ -42,20 +47,24 @@ module Clusters
         end
       end
+      def pod_name
+        install_command.pod_name
+      end
+
       def timeouted?
         Time.now.utc - app.updated_at.to_time.utc > ClusterWaitForAppInstallationWorker::TIMEOUT
       end
       def remove_installation_pod
-        helm_api.delete_pod!(install_command.pod_name)
+        helm_api.delete_pod!(pod_name)
       end
       def installation_phase
-        helm_api.status(install_command.pod_name)
+        helm_api.status(pod_name)
       end
       def installation_errors
-        helm_api.log(install_command.pod_name)
+        helm_api.log(pod_name)
       end
     end
   end

app/services/clusters/applications/schedule_installation_service.rb

@@ -10,6 +10,18 @@ module Clusters
       end
       def execute
+        application.updateable? ? schedule_upgrade : schedule_install
+      end
+
+      private
+
+      def schedule_upgrade
+        application.make_scheduled!
+
+        ClusterUpgradeAppWorker.perform_async(application.name, application.id)
+      end
+
+      def schedule_install
         application.make_scheduled!
         ClusterInstallAppWorker.perform_async(application.name, application.id)

app/services/clusters/applications/upgrade_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Applications
+    class UpgradeService < BaseHelmService
+      def execute
+        return unless app.scheduled?
+
+        begin
+          app.make_updating!
+
+          # install_command works with upgrades too
+          # as it basically does `helm upgrade --install`
+          helm_api.update(install_command)
+
+          ClusterWaitForAppInstallationWorker.perform_in(
+            ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
+        rescue Kubeclient::HttpError => e
+          log_error(e)
+          app.make_update_errored!("Kubernetes error: #{e.error_code}")
+        rescue StandardError => e
+          log_error(e)
+          app.make_update_errored!("Can't start upgrade process.")
+        end
+      end
+    end
+  end
+end

app/workers/all_queues.yml

@@ -23,6 +23,7 @@
 - cronjob:prune_web_hook_logs
 - gcp_cluster:cluster_install_app
+- gcp_cluster:cluster_upgrade_app
 - gcp_cluster:cluster_provision
 - gcp_cluster:cluster_wait_for_app_installation
 - gcp_cluster:wait_for_cluster_creation

app/workers/cluster_upgrade_app_worker.rb

+# frozen_string_literal: true
+
+class ClusterUpgradeAppWorker
+  include ApplicationWorker
+  include ClusterQueue
+  include ClusterApplications
+
+  def perform(app_name, app_id)
+    find_application(app_name, app_id) do |app|
+      Clusters::Applications::UpgradeService.new(app).execute
+    end
+  end
+end

changelogs/unreleased/use_upgrade_install_for_helm_apps.yml

+---
+title: Added ability to upgrade cluster applications
+merge_request: 24789
+author:
+type: added

lib/gitlab/kubernetes/helm/api.rb

@@ -20,14 +20,7 @@ module Gitlab
           kubeclient.create_pod(command.pod_resource)
         end
-        def update(command)
-          namespace.ensure_exists!
-
-          update_config_map(command)
-
-          delete_pod!(command.pod_name)
-          kubeclient.create_pod(command.pod_resource)
-        end
+        alias_method :update, :install
         ##
         # Returns Pod phase
@@ -62,6 +55,8 @@ module Gitlab
         def create_config_map(command)
           command.config_map_resource.tap do |config_map_resource|
+            break unless config_map_resource
+
             if config_map_exists?(config_map_resource)
               kubeclient.update_config_map(config_map_resource)
             else

lib/gitlab/kubernetes/helm/install_command.rb

@@ -42,8 +42,17 @@ module Gitlab
           'helm repo update' if repository
         end
+        # Uses `helm upgrade --install` which means we can use this for both
+        # installation and uprade of applications
         def install_command
-          command = ['helm', 'install', chart] + install_command_flags
+          command = ['helm', 'upgrade', name, chart] +
+            install_flag +
+            reset_values_flag +
+            optional_tls_flags +
+            optional_version_flag +
+            rbac_create_flag +
+            namespace_flag +
+            value_flag
           command.shelljoin
         end
@@ -56,17 +65,20 @@ module Gitlab
           postinstall.join("\n") if postinstall
         end
-        def install_command_flags
-          name_flag      = ['--name', name]
-          namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
-          value_flag     = ['-f', "/data/helm/#{name}/config/values.yaml"]
-          name_flag +
-            optional_tls_flags +
-            optional_version_flag +
-            rbac_create_flag +
-            namespace_flag +
-            value_flag
+        def install_flag
+          ['--install']
+        end
+        def reset_values_flag
+          ['--reset-values']
+        end
+
+        def value_flag
+          ['-f', "/data/helm/#{name}/config/values.yaml"]
+        end
+
+        def namespace_flag
+          ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
         end
         def rbac_create_flag

lib/gitlab/kubernetes/helm/upgrade_command.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module Kubernetes
-    module Helm
-      class UpgradeCommand
-        include BaseCommand
-        include ClientCommand
-
-        attr_reader :name, :chart, :version, :repository, :files
-
-        def initialize(name, chart:, files:, rbac:, version: nil, repository: nil)
-          @name = name
-          @chart = chart
-          @rbac = rbac
-          @version = version
-          @files = files
-          @repository = repository
-        end
-
-        def generate_script
-          super + [
-            init_command,
-            wait_for_tiller_command,
-            repository_command,
-            script_command
-          ].compact.join("\n")
-        end
-
-        def rbac?
-          @rbac
-        end
-
-        def pod_name
-          "upgrade-#{name}"
-        end
-
-        private
-
-        def script_command
-          upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \
-            " --reset-values" \
-            " --install" \
-            " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \
-            " -f /data/helm/#{name}/config/values.yaml"
-
-          "helm upgrade #{name} #{chart}#{upgrade_flags}"
-        end
-
-        def optional_version_flag
-          " --version #{version}" if version
-        end
-
-        def optional_tls_flags
-          return unless files.key?(:'ca.pem')
-
-          " --tls" \
-            " --tls-ca-cert #{files_dir}/ca.pem" \
-            " --tls-cert #{files_dir}/cert.pem" \
-            " --tls-key #{files_dir}/key.pem"
-        end
-      end
-    end
-  end
-end

locale/gitlab.pot

@@ -1554,6 +1554,9 @@ msgstr ""
 msgid "ClusterIntegration|%{boldNotice} This will add some extra resources like a load balancer, which may incur additional costs depending on the hosting provider your Kubernetes cluster is installed on. If you are using Google Kubernetes Engine, you can %{pricingLink}."
 msgstr ""
+msgid "ClusterIntegration|%{title} upgraded successfully."
+msgstr ""
+
 msgid "ClusterIntegration|API URL"
 msgstr ""
@@ -1878,6 +1881,9 @@ msgstr ""
 msgid "ClusterIntegration|Request to begin installing failed"
 msgstr ""
+msgid "ClusterIntegration|Retry upgrade"
+msgstr ""
+
 msgid "ClusterIntegration|Save changes"
 msgstr ""
@@ -1920,6 +1926,9 @@ msgstr ""
 msgid "ClusterIntegration|Something went wrong on our end."
 msgstr ""
+msgid "ClusterIntegration|Something went wrong when upgrading %{title}. Please check the logs and try again."
+msgstr ""
+
 msgid "ClusterIntegration|Something went wrong while creating your Kubernetes cluster on Google Kubernetes Engine"
 msgstr ""
@@ -1944,6 +1953,18 @@ msgstr ""
 msgid "ClusterIntegration|Token"
 msgstr ""
+msgid "ClusterIntegration|Upgrade"
+msgstr ""
+
+msgid "ClusterIntegration|Upgrade failed"
+msgstr ""
+
+msgid "ClusterIntegration|Upgraded"
+msgstr ""
+
+msgid "ClusterIntegration|Upgrading"
+msgstr ""
+
 msgid "ClusterIntegration|Validating project billing status"
 msgstr ""