Merge branch '49726-upgrade-helm-to-2-11' into 'master'

Resolve "Upgrade Helm Tiller Version Used By GitLab Managed Apps"

Closes #49726

See merge request gitlab-org/gitlab-ce!22693

app/models/clusters/concerns/application_status.rb

@@ -64,6 +64,13 @@ module Clusters
             status_reason = transition.args.first
             app_status.status_reason = status_reason if status_reason
           end
+
+          before_transition any => [:installed, :updated] do |app_status, _|
+            # When installing any application we are also performing an update
+            # of tiller (see Gitlab::Kubernetes::Helm::ClientCommand) so
+            # therefore we need to reflect that in the database.
+            app_status.cluster.application_helm.update!(version: Gitlab::Kubernetes::Helm::HELM_VERSION)
+          end
         end
       end

changelogs/unreleased/49726-upgrade-helm-to-2-11.yml

+---
+title: Upgrade helm to 2.11.0 and upgrade on every install
+merge_request: 22693
+author:
+type: added

doc/user/project/clusters/index.md

@@ -230,6 +230,10 @@ twice, which can lead to confusion during deployments.
 | [JupyterHub](http://jupyter.org/) | 11.0+ | [JupyterHub](https://jupyterhub.readthedocs.io/en/stable/) is a multi-user service for managing notebooks across a team. [Jupyter Notebooks](https://jupyter-notebook.readthedocs.io/en/latest/) provide a web-based interactive programming environment used for data analysis, visualization, and machine learning. We use [this](https://gitlab.com/gitlab-org/jupyterhub-user-image/blob/master/Dockerfile) custom Jupyter image that installs additional useful packages on top of the base Jupyter. You will also see ready-to-use DevOps Runbooks built with Nurtch's [Rubix library](https://github.com/amit1rrr/rubix). More information on creating executable runbooks can be found at [Nurtch Documentation](http://docs.nurtch.com/en/latest).  **Note**: Authentication will be enabled for any user of the GitLab server via OAuth2. HTTPS will be supported in a future release. | [jupyter/jupyterhub](https://jupyterhub.github.io/helm-chart/) |
 | [Knative](https://cloud.google.com/knative) | 0.1.2 | Knative provides a platform to create, deploy, and manage serverless workloads from a Kubernetes cluster. It is used in conjunction with, and includes [Istio](https://istio.io) to provide an external IP address for all programs hosted by Knative. You will be prompted to enter a wildcard domain where your applications will be exposed. Configure your DNS server to use the external IP address for that domain. For any application created and installed, they will be accessible as `<program_name>.<kubernetes_namespace>.<domain_name>`.  **Note**: This will require your kubernetes cluster to have RBAC enabled. | [knative/knative](https://storage.googleapis.com/triggermesh-charts)
+NOTE: **Note:**
+As of GitLab 11.6 Helm Tiller will be upgraded to the latest version supported
+by GitLab before installing any of the above applications.
+
 ## Getting the external IP address
 NOTE: **Note:**

lib/gitlab/kubernetes/helm.rb

 module Gitlab
   module Kubernetes
     module Helm
-      HELM_VERSION = '2.7.2'.freeze
+      HELM_VERSION = '2.11.0'.freeze
       KUBECTL_VERSION = '1.11.0'.freeze
       NAMESPACE = 'gitlab-managed-apps'.freeze
       SERVICE_ACCOUNT = 'tiller'.freeze

lib/gitlab/kubernetes/helm/client_command.rb

+module Gitlab
+  module Kubernetes
+    module Helm
+      module ClientCommand
+        def init_command
+          # Here we are always upgrading to the latest version of Tiller when
+          # installing an app. We ensure the helm version stored in the
+          # database is correct by also updating this after transition to
+          # :installed,:updated in Clusters::Concerns::ApplicationStatus
+          'helm init --upgrade'
+        end
+
+        def wait_for_tiller_command
+          # This is necessary to give Tiller time to restart after upgrade.
+          # Ideally we'd be able to use --wait but cannot because of
+          # https://github.com/helm/helm/issues/4855
+          'for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done'
+        end
+
+        def repository_command
+          ['helm', 'repo', 'add', name, repository].shelljoin if repository
+        end
+      end
+    end
+  end
+end

lib/gitlab/kubernetes/helm/install_command.rb

@@ -3,6 +3,7 @@ module Gitlab
     module Helm
       class InstallCommand
         include BaseCommand
+        include ClientCommand
         attr_reader :name, :files, :chart, :version, :repository, :preinstall, :postinstall
@@ -20,6 +21,7 @@ module Gitlab
         def generate_script
           super + [
             init_command,
+            wait_for_tiller_command,
             repository_command,
             repository_update_command,
             preinstall_command,
@@ -34,14 +36,6 @@ module Gitlab
         private
-        def init_command
-          'helm init --client-only'
-        end
-
-        def repository_command
-          ['helm', 'repo', 'add', name, repository].shelljoin if repository
-        end
-
         def repository_update_command
           'helm repo update' if repository
         end

lib/gitlab/kubernetes/helm/upgrade_command.rb

@@ -5,6 +5,7 @@ module Gitlab
     module Helm
       class UpgradeCommand
         include BaseCommand
+        include ClientCommand
         attr_reader :name, :chart, :version, :repository, :files
@@ -20,6 +21,7 @@ module Gitlab
         def generate_script
           super + [
             init_command,
+            wait_for_tiller_command,
             repository_command,
             script_command
           ].compact.join("\n")
@@ -35,14 +37,6 @@ module Gitlab
         private
-        def init_command
-          'helm init --client-only'
-        end
-
-        def repository_command
-          "helm repo add #{name} #{repository}" if repository
-        end
-
         def script_command
           upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \
             " --reset-values" \

spec/lib/gitlab/kubernetes/helm/install_command_spec.rb

@@ -26,7 +26,8 @@ describe Gitlab::Kubernetes::Helm::InstallCommand do
   it_behaves_like 'helm commands' do
     let(:commands) do
       <<~EOS
-      helm init --client-only
+      helm init --upgrade
+      for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
       helm repo add app-name https://repository.example.com
       helm repo update
       #{helm_install_comand}
@@ -54,7 +55,8 @@ describe Gitlab::Kubernetes::Helm::InstallCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-        helm init --client-only
+        helm init --upgrade
+        for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
         helm repo add app-name https://repository.example.com
         helm repo update
         #{helm_install_command}
@@ -84,7 +86,8 @@ describe Gitlab::Kubernetes::Helm::InstallCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-        helm init --client-only
+        helm init --upgrade
+        for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
         #{helm_install_command}
         EOS
       end
@@ -111,7 +114,8 @@ describe Gitlab::Kubernetes::Helm::InstallCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-        helm init --client-only
+        helm init --upgrade
+        for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
         helm repo add app-name https://repository.example.com
         helm repo update
         #{helm_install_command}
@@ -134,7 +138,8 @@ describe Gitlab::Kubernetes::Helm::InstallCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-        helm init --client-only
+        helm init --upgrade
+        for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
         helm repo add app-name https://repository.example.com
         helm repo update
         #{helm_install_command}
@@ -157,7 +162,8 @@ describe Gitlab::Kubernetes::Helm::InstallCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-        helm init --client-only
+        helm init --upgrade
+        for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
         helm repo add app-name https://repository.example.com
         helm repo update
         #{helm_install_command}
@@ -182,7 +188,8 @@ describe Gitlab::Kubernetes::Helm::InstallCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-        helm init --client-only
+        helm init --upgrade
+        for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
         helm repo add app-name https://repository.example.com
         helm repo update
         #{helm_install_command}

spec/lib/gitlab/kubernetes/helm/pod_spec.rb

@@ -30,7 +30,7 @@ describe Gitlab::Kubernetes::Helm::Pod do
       it 'should generate the appropriate specifications for the container' do
         container = subject.generate.spec.containers.first
         expect(container.name).to eq('helm')
-        expect(container.image).to eq('registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/2.7.2-kube-1.11.0')
+        expect(container.image).to eq('registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/2.11.0-kube-1.11.0')
         expect(container.env.count).to eq(3)
         expect(container.env.map(&:name)).to match_array([:HELM_VERSION, :TILLER_NAMESPACE, :COMMAND_SCRIPT])
         expect(container.command).to match_array(["/bin/sh"])

spec/lib/gitlab/kubernetes/helm/upgrade_command_spec.rb

@@ -21,7 +21,8 @@ describe Gitlab::Kubernetes::Helm::UpgradeCommand do
   it_behaves_like 'helm commands' do
     let(:commands) do
       <<~EOS
-         helm init --client-only
+         helm init --upgrade
+         for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
          helm upgrade #{application.name} #{application.chart} --tls --tls-ca-cert /data/helm/#{application.name}/config/ca.pem --tls-cert /data/helm/#{application.name}/config/cert.pem --tls-key /data/helm/#{application.name}/config/key.pem --reset-values --install --namespace #{namespace} -f /data/helm/#{application.name}/config/values.yaml
       EOS
     end
@@ -33,7 +34,8 @@ describe Gitlab::Kubernetes::Helm::UpgradeCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-         helm init --client-only
+         helm init --upgrade
+         for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
          helm upgrade #{application.name} #{application.chart} --tls --tls-ca-cert /data/helm/#{application.name}/config/ca.pem --tls-cert /data/helm/#{application.name}/config/cert.pem --tls-key /data/helm/#{application.name}/config/key.pem --reset-values --install --namespace #{namespace} -f /data/helm/#{application.name}/config/values.yaml
         EOS
       end
@@ -56,7 +58,8 @@ describe Gitlab::Kubernetes::Helm::UpgradeCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-           helm init --client-only
+           helm init --upgrade
+           for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
            helm repo add #{application.name} #{application.repository}
            helm upgrade #{application.name} #{application.chart} --tls --tls-ca-cert /data/helm/#{application.name}/config/ca.pem --tls-cert /data/helm/#{application.name}/config/cert.pem --tls-key /data/helm/#{application.name}/config/key.pem --reset-values --install --namespace #{namespace} -f /data/helm/#{application.name}/config/values.yaml
         EOS
@@ -70,7 +73,8 @@ describe Gitlab::Kubernetes::Helm::UpgradeCommand do
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
-         helm init --client-only
+         helm init --upgrade
+         for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
          helm upgrade #{application.name} #{application.chart} --reset-values --install --namespace #{namespace} -f /data/helm/#{application.name}/config/values.yaml
         EOS
       end

spec/models/clusters/applications/prometheus_spec.rb

@@ -35,7 +35,7 @@ describe Clusters::Applications::Prometheus do
   describe 'transition to installed' do
     let(:project) { create(:project) }
-    let(:cluster) { create(:cluster, projects: [project]) }
+    let(:cluster) { create(:cluster, :with_installed_helm, projects: [project]) }
     let(:prometheus_service) { double('prometheus_service') }
     subject { create(:clusters_applications_prometheus, :installing, cluster: cluster) }

spec/support/shared_examples/models/cluster_application_status_shared_examples.rb

@@ -44,10 +44,40 @@ shared_examples 'cluster application status specs' do |application_name|
       subject { create(application_name, :installing) }
       it 'is installed' do
-        subject.make_installed
+        subject.make_installed!
         expect(subject).to be_installed
       end
+
+      it 'updates helm version' do
+        subject.cluster.application_helm.update!(version: '1.2.3')
+
+        subject.make_installed!
+
+        subject.cluster.application_helm.reload
+
+        expect(subject.cluster.application_helm.version).to eq(Gitlab::Kubernetes::Helm::HELM_VERSION)
+      end
+    end
+
+    describe '#make_updated' do
+      subject { create(application_name, :updating) }
+
+      it 'is updated' do
+        subject.make_updated!
+
+        expect(subject).to be_updated
+      end
+
+      it 'updates helm version' do
+        subject.cluster.application_helm.update!(version: '1.2.3')
+
+        subject.make_updated!
+
+        subject.cluster.application_helm.reload
+
+        expect(subject.cluster.application_helm.version).to eq(Gitlab::Kubernetes::Helm::HELM_VERSION)
+      end
     end
     describe '#make_errored' do