Merge branch '59276-more-k8s-logging' into 'master'

Resolve "Improvements of kubernetes.log"

Closes #59276

See merge request gitlab-org/gitlab-ce!26895

app/services/clusters/applications/base_helm_service.rb

@@ -16,6 +16,7 @@ def log_error(error)
           error_code: error.respond_to?(:error_code) ? error.error_code : nil,
           service: self.class.name,
           app_id: app.id,
+          app_name: app.name,
           project_ids: app.cluster.project_ids,
           group_ids: app.cluster.group_ids
         }
@@ -30,6 +31,19 @@ def log_error(error)
         Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
       end
+      def log_event(event)
+        meta = {
+          service: self.class.name,
+          app_id: app.id,
+          app_name: app.name,
+          project_ids: app.cluster.project_ids,
+          group_ids: app.cluster.group_ids,
+          event: event
+        }
+
+        logger.info(meta)
+      end
+
       def logger
         @logger ||= Gitlab::Kubernetes::Logger.build
       end

app/services/clusters/applications/install_service.rb

@@ -7,8 +7,10 @@ def execute
         return unless app.scheduled?
         app.make_installing!
+        log_event(:begin_install)
         helm_api.install(install_command)
+        log_event(:schedule_wait_for_installation)
         ClusterWaitForAppInstallationWorker.perform_in(
           ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
       rescue Kubeclient::HttpError => e

app/services/clusters/applications/patch_service.rb

@@ -8,8 +8,10 @@ def execute
         app.make_updating!
+        log_event(:begin_patch)
         helm_api.update(update_command)
+        log_event(:schedule_wait_for_patch)
         ClusterWaitForAppInstallationWorker.perform_in(
           ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
       rescue Kubeclient::HttpError => e

app/services/clusters/applications/upgrade_service.rb

@@ -9,10 +9,12 @@ def execute
         begin
           app.make_updating!
+          log_event(:begin_upgrade)
           # install_command works with upgrades too
           # as it basically does `helm upgrade --install`
           helm_api.update(install_command)
+          log_event(:schedule_wait_for_upgrade)
           ClusterWaitForAppInstallationWorker.perform_in(
             ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
         rescue Kubeclient::HttpError => e

doc/user/project/clusters/index.md

@@ -573,7 +573,7 @@ However, sometimes GitLab can not create them. In such instances, your job will
 This job failed because the necessary resources were not successfully created.
 ```
-To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#sidekiqlog).
+To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#kuberneteslog).
 Common reasons for failure include:

lib/gitlab/kubernetes/namespace.rb

@@ -19,11 +19,40 @@ def exists?
       def create!
         resource = ::Kubeclient::Resource.new(metadata: { name: name })
+        log_event(:begin_create)
         @client.create_namespace(resource)
       end
       def ensure_exists!
         exists? || create!
+      rescue ::Kubeclient::HttpError => error
+        log_create_failed(error)
+        raise
+      end
+
+      private
+
+      def log_create_failed(error)
+        logger.error({
+          exception: error.class.name,
+          status_code: error.error_code,
+          namespace: name,
+          class_name: self.class.name,
+          event: :failed_to_create_namespace,
+          message: error.message
+        })
+      end
+
+      def log_event(event)
+        logger.info(
+          namespace: name,
+          class_name: self.class.name,
+          event: event
+        )
+      end
+
+      def logger
+        @logger ||= Gitlab::Kubernetes::Logger.build
       end
     end
   end

spec/lib/gitlab/kubernetes/namespace_spec.rb

@@ -62,5 +62,32 @@
       subject.ensure_exists!
     end
+
+    context 'when client errors' do
+      let(:exception) { Kubeclient::HttpError.new(500, 'system failure', nil) }
+
+      before do
+        allow(client).to receive(:get_namespace).with(name).once.and_raise(exception)
+      end
+
+      it 'raises the exception' do
+        expect { subject.ensure_exists! }.to raise_error(exception)
+      end
+
+      it 'logs the error' do
+        expect(subject.send(:logger)).to receive(:error).with(
+          hash_including(
+            exception: 'Kubeclient::HttpError',
+            status_code: 500,
+            namespace: 'a_namespace',
+            class_name: 'Gitlab::Kubernetes::Namespace',
+            event: :failed_to_create_namespace,
+            message: 'system failure'
+          )
+        )
+
+        expect { subject.ensure_exists! }.to raise_error(exception)
+      end
+    end
   end
 end

spec/support/shared_examples/services/base_helm_service_shared_examples.rb

@@ -20,7 +20,7 @@
   end
   it 'logs into kubernetes.log and Sentry' do
-    expect(service.send(:logger)).to receive(:error).with(logger_hash)
+    expect(service.send(:logger)).to receive(:error).with(hash_including(logger_hash))
     expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
       error,