Merge branch '48098-mutual-auth-cluster-applications' into 'master'

Resolve "Mutual SSL Auth For Helm TIller"

Closes #48098

See merge request gitlab-org/gitlab-ce!20801

spec/features/projects/clusters/applications_spec.rb

@@ -46,12 +46,14 @@ describe 'Clusters Applications', :js do
           end
         end
-        it 'he sees status transition' do
+        it 'they see status transition' do
           page.within('.js-cluster-application-row-helm') do
             # FE sends request and gets the response, then the buttons is "Install"
             expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true')
             expect(page).to have_css('.js-cluster-application-install-button', exact_text: 'Install')
+            wait_until_helm_created!
+
             Clusters::Cluster.last.application_helm.make_installing!
             # FE starts polling and update the buttons to "Installing"
@@ -83,7 +85,7 @@ describe 'Clusters Applications', :js do
             end
           end
-          it 'he sees status transition' do
+          it 'they see status transition' do
             page.within('.js-cluster-application-row-ingress') do
               # FE sends request and gets the response, then the buttons is "Install"
               expect(page).to have_css('.js-cluster-application-install-button[disabled]')
@@ -116,4 +118,14 @@ describe 'Clusters Applications', :js do
       end
     end
   end
+
+  def wait_until_helm_created!
+    retries = 0
+
+    while Clusters::Cluster.last.application_helm.nil?
+      raise "Timed out waiting for helm application to be created in DB" if (retries += 1) > 3
+
+      sleep(1)
+    end
+  end
 end

spec/lib/gitlab/kubernetes/config_map_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Gitlab::Kubernetes::ConfigMap do
   let(:kubeclient) { double('kubernetes client') }
   let(:application) { create(:clusters_applications_prometheus) }
-  let(:config_map) { described_class.new(application.name, application.values) }
+  let(:config_map) { described_class.new(application.name, application.files) }
   let(:namespace) { Gitlab::Kubernetes::Helm::NAMESPACE }
   let(:metadata) do
@@ -15,7 +15,7 @@ describe Gitlab::Kubernetes::ConfigMap do
   end
   describe '#generate' do
-    let(:resource) { ::Kubeclient::Resource.new(metadata: metadata, data: { values: application.values }) }
+    let(:resource) { ::Kubeclient::Resource.new(metadata: metadata, data: application.files) }
     subject { config_map.generate }
     it 'should build a Kubeclient Resource' do

spec/lib/gitlab/kubernetes/helm/api_spec.rb

@@ -39,7 +39,7 @@ describe Gitlab::Kubernetes::Helm::Api do
     end
     context 'with a ConfigMap' do
-      let(:resource) { Gitlab::Kubernetes::ConfigMap.new(application.name, application.values).generate }
+      let(:resource) { Gitlab::Kubernetes::ConfigMap.new(application.name, application.files).generate }
       it 'creates a ConfigMap on kubeclient' do
         expect(client).to receive(:create_config_map).with(resource).once

spec/lib/gitlab/kubernetes/helm/base_command_spec.rb

@@ -2,7 +2,25 @@ require 'spec_helper'
 describe Gitlab::Kubernetes::Helm::BaseCommand do
   let(:application) { create(:clusters_applications_helm) }
-  let(:base_command) { described_class.new(application.name) }
+  let(:test_class) do
+    Class.new do
+      include Gitlab::Kubernetes::Helm::BaseCommand
+
+      def name
+        "test-class-name"
+      end
+
+      def files
+        {
+          some: 'value'
+        }
+      end
+    end
+  end
+
+  let(:base_command) do
+    test_class.new
+  end
   subject { base_command }
@@ -18,15 +36,9 @@ describe Gitlab::Kubernetes::Helm::BaseCommand do
     end
   end
-  describe '#config_map?' do
-    subject { base_command.config_map? }
-
-    it { is_expected.to be_falsy }
-  end
-
   describe '#pod_name' do
     subject { base_command.pod_name }
-    it { is_expected.to eq('install-helm') }
+    it { is_expected.to eq('install-test-class-name') }
   end
 end

spec/lib/gitlab/kubernetes/helm/certificate_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Kubernetes::Helm::Certificate do
+  describe '.generate_root' do
+    subject { described_class.generate_root }
+
+    it 'should generate a root CA that expires a long way in the future' do
+      expect(subject.cert.not_after).to be > 999.years.from_now
+    end
+  end
+
+  describe '#issue' do
+    subject { described_class.generate_root.issue }
+
+    it 'should generate a cert that expires soon' do
+      expect(subject.cert.not_after).to be < 60.minutes.from_now
+    end
+
+    context 'passing in INFINITE_EXPIRY' do
+      subject { described_class.generate_root.issue(expires_in: described_class::INFINITE_EXPIRY) }
+
+      it 'should generate a cert that expires a long way in the future' do
+        expect(subject.cert.not_after).to be > 999.years.from_now
+      end
+    end
+  end
+end

spec/lib/gitlab/kubernetes/helm/init_command_spec.rb

@@ -2,9 +2,9 @@ require 'spec_helper'
 describe Gitlab::Kubernetes::Helm::InitCommand do
   let(:application) { create(:clusters_applications_helm) }
-  let(:commands) { 'helm init >/dev/null' }
-  subject { described_class.new(application.name) }
+  let(:commands) { 'helm init --tiller-tls --tiller-tls-verify --tls-ca-cert /data/helm/helm/config/ca.pem --tiller-tls-cert /data/helm/helm/config/cert.pem --tiller-tls-key /data/helm/helm/config/key.pem >/dev/null' }
+  subject { described_class.new(name: application.name, files: {}) }
   it_behaves_like 'helm commands'
 end

spec/lib/gitlab/kubernetes/helm/install_command_spec.rb

 require 'rails_helper'
 describe Gitlab::Kubernetes::Helm::InstallCommand do
-  let(:application) { create(:clusters_applications_prometheus) }
-  let(:namespace) { Gitlab::Kubernetes::Helm::NAMESPACE }
-  let(:install_command) { application.install_command }
+  let(:files) { { 'ca.pem': 'some file content' } }
+  let(:repository) { 'https://repository.example.com' }
+  let(:version) { '1.2.3' }
+
+  let(:install_command) do
+    described_class.new(
+      name: 'app-name',
+      chart: 'chart-name',
+      files: files,
+      version: version, repository: repository
+    )
+  end
   subject { install_command }
-  context 'for ingress' do
-    let(:application) { create(:clusters_applications_ingress) }
-
-    it_behaves_like 'helm commands' do
-      let(:commands) do
-        <<~EOS
-         helm init --client-only >/dev/null
-         helm install #{application.chart} --name #{application.name} --namespace #{namespace} -f /data/helm/#{application.name}/config/values.yaml >/dev/null
-        EOS
-      end
+  it_behaves_like 'helm commands' do
+    let(:commands) do
+      <<~EOS
+      helm init --client-only >/dev/null
+      helm repo add app-name https://repository.example.com
+      helm install --tls --tls-ca-cert /data/helm/app-name/config/ca.pem --tls-cert /data/helm/app-name/config/cert.pem --tls-key /data/helm/app-name/config/key.pem chart-name --name app-name --version 1.2.3 --namespace gitlab-managed-apps -f /data/helm/app-name/config/values.yaml >/dev/null
+      EOS
     end
   end
-  context 'for prometheus' do
-    let(:application) { create(:clusters_applications_prometheus) }
+  context 'when there is no repository' do
+    let(:repository) { nil }
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
          helm init --client-only >/dev/null
-         helm install #{application.chart} --name #{application.name} --version #{application.version} --namespace #{namespace} -f /data/helm/#{application.name}/config/values.yaml >/dev/null
+         helm install --tls --tls-ca-cert /data/helm/app-name/config/ca.pem --tls-cert /data/helm/app-name/config/cert.pem --tls-key /data/helm/app-name/config/key.pem chart-name --name app-name --version 1.2.3 --namespace gitlab-managed-apps -f /data/helm/app-name/config/values.yaml >/dev/null
         EOS
       end
     end
   end
-  context 'for runner' do
-    let(:ci_runner) { create(:ci_runner) }
-    let(:application) { create(:clusters_applications_runner, runner: ci_runner) }
+  context 'when there is no ca.pem file' do
+    let(:files) { { 'file.txt': 'some content' } }
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
          helm init --client-only >/dev/null
-         helm repo add #{application.name} #{application.repository}
-         helm install #{application.chart} --name #{application.name} --namespace #{namespace} -f /data/helm/#{application.name}/config/values.yaml >/dev/null
+         helm repo add app-name https://repository.example.com
+         helm install chart-name --name app-name --version 1.2.3 --namespace gitlab-managed-apps -f /data/helm/app-name/config/values.yaml >/dev/null
         EOS
       end
     end
   end
-  context 'for jupyter' do
-    let(:application) { create(:clusters_applications_jupyter) }
+  context 'when there is no version' do
+    let(:version) { nil }
     it_behaves_like 'helm commands' do
       let(:commands) do
         <<~EOS
          helm init --client-only >/dev/null
-         helm repo add #{application.name} #{application.repository}
-         helm install #{application.chart} --name #{application.name} --namespace #{namespace} -f /data/helm/#{application.name}/config/values.yaml >/dev/null
+         helm repo add app-name https://repository.example.com
+         helm install --tls --tls-ca-cert /data/helm/app-name/config/ca.pem --tls-cert /data/helm/app-name/config/cert.pem --tls-key /data/helm/app-name/config/key.pem chart-name --name app-name --namespace gitlab-managed-apps -f /data/helm/app-name/config/values.yaml >/dev/null
         EOS
       end
     end
   end
-  describe '#config_map?' do
-    subject { install_command.config_map? }
-
-    it { is_expected.to be_truthy }
-  end
-
   describe '#config_map_resource' do
     let(:metadata) do
       {
-        name: "values-content-configuration-#{application.name}",
-        namespace: namespace,
-        labels: { name: "values-content-configuration-#{application.name}" }
+        name: "values-content-configuration-app-name",
+        namespace: 'gitlab-managed-apps',
+        labels: { name: "values-content-configuration-app-name" }
       }
     end
-    let(:resource) { ::Kubeclient::Resource.new(metadata: metadata, data: { values: application.values }) }
+    let(:resource) { ::Kubeclient::Resource.new(metadata: metadata, data: files) }
     subject { install_command.config_map_resource }

spec/lib/gitlab/kubernetes/helm/pod_spec.rb

@@ -2,14 +2,13 @@ require 'rails_helper'
 describe Gitlab::Kubernetes::Helm::Pod do
   describe '#generate' do
-    let(:cluster) { create(:cluster) }
-    let(:app) {  create(:clusters_applications_prometheus, cluster: cluster) }
+    let(:app) {  create(:clusters_applications_prometheus) }
     let(:command) {  app.install_command }
     let(:namespace) { Gitlab::Kubernetes::Helm::NAMESPACE }
     subject { described_class.new(command, namespace) }
-    shared_examples 'helm pod' do
+    context 'with a command' do
       it 'should generate a Kubeclient::Resource' do
         expect(subject.generate).to be_a_kind_of(Kubeclient::Resource)
       end
@@ -41,10 +40,6 @@ describe Gitlab::Kubernetes::Helm::Pod do
         spec = subject.generate.spec
         expect(spec.restartPolicy).to eq('Never')
       end
-    end
-
-    context 'with a install command' do
-      it_behaves_like 'helm pod'
       it 'should include volumes for the container' do
         container = subject.generate.spec.containers.first
@@ -60,24 +55,8 @@ describe Gitlab::Kubernetes::Helm::Pod do
       it 'should mount configMap specification in the volume' do
         volume = subject.generate.spec.volumes.first
         expect(volume.configMap['name']).to eq("values-content-configuration-#{app.name}")
-        expect(volume.configMap['items'].first['key']).to eq('values')
-        expect(volume.configMap['items'].first['path']).to eq('values.yaml')
-      end
-    end
-
-    context 'with a init command' do
-      let(:app) { create(:clusters_applications_helm, cluster: cluster) }
-
-      it_behaves_like 'helm pod'
-
-      it 'should not include volumeMounts inside the container' do
-        container = subject.generate.spec.containers.first
-        expect(container.volumeMounts).to be_nil
-      end
-
-      it 'should not a volume inside the specification' do
-        spec = subject.generate.spec
-        expect(spec.volumes).to be_nil
+        expect(volume.configMap['items'].first['key']).to eq(:'values.yaml')
+        expect(volume.configMap['items'].first['path']).to eq(:'values.yaml')
       end
     end
   end

spec/models/clusters/applications/helm_spec.rb

@@ -6,13 +6,24 @@ describe Clusters::Applications::Helm do
   describe '.installed' do
     subject { described_class.installed }
-    let!(:cluster) { create(:clusters_applications_helm, :installed) }
+    let!(:installed_cluster) { create(:clusters_applications_helm, :installed) }
     before do
       create(:clusters_applications_helm, :errored)
     end
-    it { is_expected.to contain_exactly(cluster) }
+    it { is_expected.to contain_exactly(installed_cluster) }
+  end
+
+  describe '#issue_client_cert' do
+    let(:application) { create(:clusters_applications_helm) }
+    subject { application.issue_client_cert }
+
+    it 'returns a new cert' do
+      is_expected.to be_kind_of(Gitlab::Kubernetes::Helm::Certificate)
+      expect(subject.cert_string).not_to eq(application.ca_cert)
+      expect(subject.key_string).not_to eq(application.ca_key)
+    end
   end
   describe '#install_command' do
@@ -25,5 +36,16 @@ describe Clusters::Applications::Helm do
     it 'should be initialized with 1 arguments' do
       expect(subject.name).to eq('helm')
     end
+
+    it 'should have cert files' do
+      expect(subject.files[:'ca.pem']).to be_present
+      expect(subject.files[:'ca.pem']).to eq(helm.ca_cert)
+
+      expect(subject.files[:'cert.pem']).to be_present
+      expect(subject.files[:'key.pem']).to be_present
+
+      cert = OpenSSL::X509::Certificate.new(subject.files[:'cert.pem'])
+      expect(cert.not_after).to be > 999.years.from_now
+    end
   end
 end

spec/models/clusters/applications/ingress_spec.rb

@@ -74,18 +74,43 @@ describe Clusters::Applications::Ingress do
       expect(subject.name).to eq('ingress')
       expect(subject.chart).to eq('stable/nginx-ingress')
       expect(subject.version).to be_nil
-      expect(subject.values).to eq(ingress.values)
+      expect(subject.files).to eq(ingress.files)
     end
   end
-  describe '#values' do
-    subject { ingress.values }
-    it 'should include ingress valid keys' do
-      is_expected.to include('image')
-      is_expected.to include('repository')
-      is_expected.to include('stats')
-      is_expected.to include('podAnnotations')
+  describe '#files' do
+    let(:application) { ingress }
+    subject { application.files }
+    let(:values) { subject[:'values.yaml'] }
+    it 'should include ingress valid keys in values' do
+      expect(values).to include('image')
+      expect(values).to include('repository')
+      expect(values).to include('stats')
+      expect(values).to include('podAnnotations')
+    end
+
+    context 'when the helm application does not have a ca_cert' do
+      before do
+        application.cluster.application_helm.ca_cert = nil
+      end
+
+      it 'should not include cert files' do
+        expect(subject[:'ca.pem']).not_to be_present
+        expect(subject[:'cert.pem']).not_to be_present
+        expect(subject[:'key.pem']).not_to be_present
+      end
+    end
+
+    it 'should include cert files' do
+      expect(subject[:'ca.pem']).to be_present
+      expect(subject[:'ca.pem']).to eq(application.cluster.application_helm.ca_cert)
+
+      expect(subject[:'cert.pem']).to be_present
+      expect(subject[:'key.pem']).to be_present
+
+      cert = OpenSSL::X509::Certificate.new(subject[:'cert.pem'])
+      expect(cert.not_after).to be < 60.minutes.from_now
     end
   end
 end

spec/models/clusters/applications/jupyter_spec.rb

@@ -38,23 +38,46 @@ describe Clusters::Applications::Jupyter do
       expect(subject.chart).to eq('jupyter/jupyterhub')
       expect(subject.version).to be_nil
       expect(subject.repository).to eq('https://jupyterhub.github.io/helm-chart/')
-      expect(subject.values).to eq(jupyter.values)
+      expect(subject.files).to eq(jupyter.files)
     end
   end
-  describe '#values' do
-    let(:jupyter) { create(:clusters_applications_jupyter) }
-    subject { jupyter.values }
+  describe '#files' do
+    let(:application) { create(:clusters_applications_jupyter) }
+    subject { application.files }
+    let(:values) { subject[:'values.yaml'] }
+    it 'should include cert files' do
+      expect(subject[:'ca.pem']).to be_present
+      expect(subject[:'ca.pem']).to eq(application.cluster.application_helm.ca_cert)
+
+      expect(subject[:'cert.pem']).to be_present
+      expect(subject[:'key.pem']).to be_present
+
+      cert = OpenSSL::X509::Certificate.new(subject[:'cert.pem'])
+      expect(cert.not_after).to be < 60.minutes.from_now
+    end
+
+    context 'when the helm application does not have a ca_cert' do
+      before do
+        application.cluster.application_helm.ca_cert = nil
+      end
+
+      it 'should not include cert files' do
+        expect(subject[:'ca.pem']).not_to be_present
+        expect(subject[:'cert.pem']).not_to be_present
+        expect(subject[:'key.pem']).not_to be_present
+      end
+    end
     it 'should include valid values' do
-      is_expected.to include('ingress')
-      is_expected.to include('hub')
-      is_expected.to include('rbac')
-      is_expected.to include('proxy')
-      is_expected.to include('auth')
-      is_expected.to include("clientId: #{jupyter.oauth_application.uid}")
-      is_expected.to include("callbackUrl: #{jupyter.callback_url}")
+      expect(values).to include('ingress')
+      expect(values).to include('hub')
+      expect(values).to include('rbac')
+      expect(values).to include('proxy')
+      expect(values).to include('auth')
+      expect(values).to match(/clientId: '?#{application.oauth_application.uid}/)
+      expect(values).to match(/callbackUrl: '?#{application.callback_url}/)
     end
   end
 end

spec/models/clusters/applications/prometheus_spec.rb

@@ -153,21 +153,44 @@ describe Clusters::Applications::Prometheus do
       expect(command.name).to eq('prometheus')
       expect(command.chart).to eq('stable/prometheus')
       expect(command.version).to eq('6.7.3')
-      expect(command.values).to eq(prometheus.values)
+      expect(command.files).to eq(prometheus.files)
     end
   end
-  describe '#values' do
-    let(:prometheus) { create(:clusters_applications_prometheus) }
-    subject { prometheus.values }
+  describe '#files' do
+    let(:application) { create(:clusters_applications_prometheus) }
+    subject { application.files }
+    let(:values) { subject[:'values.yaml'] }
+
+    it 'should include cert files' do
+      expect(subject[:'ca.pem']).to be_present
+      expect(subject[:'ca.pem']).to eq(application.cluster.application_helm.ca_cert)
+
+      expect(subject[:'cert.pem']).to be_present
+      expect(subject[:'key.pem']).to be_present
+
+      cert = OpenSSL::X509::Certificate.new(subject[:'cert.pem'])
+      expect(cert.not_after).to be < 60.minutes.from_now
+    end
+    context 'when the helm application does not have a ca_cert' do
+      before do
+        application.cluster.application_helm.ca_cert = nil
+      end
+
+      it 'should not include cert files' do
+        expect(subject[:'ca.pem']).not_to be_present
+        expect(subject[:'cert.pem']).not_to be_present
+        expect(subject[:'key.pem']).not_to be_present
+      end
+    end
     it 'should include prometheus valid values' do
-      is_expected.to include('alertmanager')
-      is_expected.to include('kubeStateMetrics')
-      is_expected.to include('nodeExporter')
-      is_expected.to include('pushgateway')
-      is_expected.to include('serverFiles')
+      expect(values).to include('alertmanager')
+      expect(values).to include('kubeStateMetrics')
+      expect(values).to include('nodeExporter')
+      expect(values).to include('pushgateway')
+      expect(values).to include('serverFiles')
     end
   end
 end

spec/models/clusters/applications/runner_spec.rb

@@ -33,31 +33,55 @@ describe Clusters::Applications::Runner do
       expect(subject.chart).to eq('runner/gitlab-runner')
       expect(subject.version).to be_nil
       expect(subject.repository).to eq('https://charts.gitlab.io')
-      expect(subject.values).to eq(gitlab_runner.values)
+      expect(subject.files).to eq(gitlab_runner.files)
     end
   end
-  describe '#values' do
-    let(:gitlab_runner) { create(:clusters_applications_runner, runner: ci_runner) }
-    subject { gitlab_runner.values }
+  describe '#files' do
+    let(:application) { create(:clusters_applications_runner, runner: ci_runner) }
+
+    subject { application.files }
+    let(:values) { subject[:'values.yaml'] }
+
+    it 'should include cert files' do
+      expect(subject[:'ca.pem']).to be_present
+      expect(subject[:'ca.pem']).to eq(application.cluster.application_helm.ca_cert)
+
+      expect(subject[:'cert.pem']).to be_present
+      expect(subject[:'key.pem']).to be_present
+
+      cert = OpenSSL::X509::Certificate.new(subject[:'cert.pem'])
+      expect(cert.not_after).to be < 60.minutes.from_now
+    end
+    context 'when the helm application does not have a ca_cert' do
+      before do
+        application.cluster.application_helm.ca_cert = nil
+      end
+
+      it 'should not include cert files' do
+        expect(subject[:'ca.pem']).not_to be_present
+        expect(subject[:'cert.pem']).not_to be_present
+        expect(subject[:'key.pem']).not_to be_present
+      end
+    end
     it 'should include runner valid values' do
-      is_expected.to include('concurrent')
-      is_expected.to include('checkInterval')
-      is_expected.to include('rbac')
-      is_expected.to include('runners')
-      is_expected.to include('privileged: true')
-      is_expected.to include('image: ubuntu:16.04')
-      is_expected.to include('resources')
-      is_expected.to include("runnerToken: #{ci_runner.token}")
-      is_expected.to include("gitlabUrl: #{Gitlab::Routing.url_helpers.root_url}")
+      expect(values).to include('concurrent')
+      expect(values).to include('checkInterval')
+      expect(values).to include('rbac')
+      expect(values).to include('runners')
+      expect(values).to include('privileged: true')
+      expect(values).to include('image: ubuntu:16.04')
+      expect(values).to include('resources')
+      expect(values).to match(/runnerToken: '?#{ci_runner.token}/)
+      expect(values).to match(/gitlabUrl: '?#{Gitlab::Routing.url_helpers.root_url}/)
     end
     context 'without a runner' do
       let(:project) { create(:project) }
-      let(:cluster) { create(:cluster, projects: [project]) }
-      let(:gitlab_runner) { create(:clusters_applications_runner, cluster: cluster) }
+      let(:cluster) { create(:cluster, :with_installed_helm, projects: [project]) }
+      let(:application) { create(:clusters_applications_runner, cluster: cluster) }
       it 'creates a runner' do
         expect do
@@ -66,18 +90,18 @@ describe Clusters::Applications::Runner do
       end
       it 'uses the new runner token' do
-        expect(subject).to include("runnerToken: #{gitlab_runner.reload.runner.token}")
+        expect(values).to match(/runnerToken: '?#{application.reload.runner.token}/)
       end
       it 'assigns the new runner to runner' do
         subject
-        expect(gitlab_runner.reload.runner).to be_project_type
+        expect(application.reload.runner).to be_project_type
       end
     end
     context 'with duplicated values on vendor/runner/values.yaml' do
-      let(:values) do
+      let(:stub_values) do
         {
           "concurrent" => 4,
           "checkInterval" => 3,
@@ -96,11 +120,11 @@ describe Clusters::Applications::Runner do
       end
       before do
-        allow(gitlab_runner).to receive(:chart_values).and_return(values)
+        allow(application).to receive(:chart_values).and_return(stub_values)
       end
       it 'should overwrite values.yaml' do
-        is_expected.to include("privileged: #{gitlab_runner.privileged}")
+        expect(values).to match(/privileged: '?#{application.privileged}/)
       end
     end
   end

spec/services/clusters/applications/install_service_spec.rb

@@ -47,7 +47,7 @@ describe Clusters::Applications::InstallService do
     end
     context 'when application cannot be persisted' do
-      let(:application) { build(:clusters_applications_helm, :scheduled) }
+      let(:application) { create(:clusters_applications_helm, :scheduled) }
       it 'make the application errored' do
         expect(application).to receive(:make_installing!).once.and_raise(ActiveRecord::RecordInvalid)