Skip to content
Snippets Groups Projects
Commit d36ebb34 authored by Hordur Freyr Yngvason's avatar Hordur Freyr Yngvason Committed by Thong Kuah
Browse files

Add Elastic Stack cluster integration

parent 608211ed
No related branches found
No related tags found
No related merge requests found
Showing
with 295 additions and 93 deletions
Loading
Loading
@@ -62,6 +62,7 @@ def cluster_status
def show
if params[:tab] == 'integrations'
@prometheus_integration = Clusters::IntegrationPresenter.new(@cluster.find_or_build_integration_prometheus)
@elastic_stack_integration = Clusters::IntegrationPresenter.new(@cluster.find_or_build_integration_elastic_stack)
end
end
 
Loading
Loading
Loading
Loading
@@ -24,7 +24,7 @@ def clusterable
end
 
def cluster_integration_params
params.require(:integration).permit(:application_type, :enabled)
params.permit(integration: [:enabled, :application_type]).require(:integration)
end
 
def cluster
Loading
Loading
Loading
Loading
@@ -3,9 +3,9 @@
module Clusters
module Applications
class ElasticStack < ApplicationRecord
VERSION = '3.0.0'
include ::Clusters::Concerns::ElasticsearchClient
 
ELASTICSEARCH_PORT = 9200
VERSION = '3.0.0'
 
self.table_name = 'clusters_applications_elastic_stacks'
 
Loading
Loading
@@ -13,10 +13,23 @@ class ElasticStack < ApplicationRecord
include ::Clusters::Concerns::ApplicationStatus
include ::Clusters::Concerns::ApplicationVersion
include ::Clusters::Concerns::ApplicationData
include ::Gitlab::Utils::StrongMemoize
 
default_value_for :version, VERSION
 
after_destroy do
cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil)
end
state_machine :status do
after_transition any => [:installed] do |application|
application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: true, chart_version: application.version)
end
after_transition any => [:uninstalled] do |application|
application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil)
end
end
def chart
'elastic-stack/elastic-stack'
end
Loading
Loading
@@ -51,31 +64,6 @@ def files
super.merge('wait-for-elasticsearch.sh': File.read("#{Rails.root}/vendor/elastic_stack/wait-for-elasticsearch.sh"))
end
 
def elasticsearch_client(timeout: nil)
strong_memoize(:elasticsearch_client) do
next unless kube_client
proxy_url = kube_client.proxy_url('service', service_name, ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE)
Elasticsearch::Client.new(url: proxy_url) do |faraday|
# ensures headers containing auth data are appended to original client options
faraday.headers.merge!(kube_client.headers)
# ensure TLS certs are properly verified
faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl]
faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store]
faraday.options.timeout = timeout unless timeout.nil?
end
rescue Kubeclient::HttpError => error
# If users have mistakenly set parameters or removed the depended clusters,
# `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
# We check for a nil client in downstream use and behaviour is equivalent to an empty state
log_exception(error, :failed_to_create_elasticsearch_client)
nil
end
end
def chart_above_v2?
Gem::Version.new(version) >= Gem::Version.new('2.0.0')
end
Loading
Loading
@@ -106,10 +94,6 @@ def post_delete_script
]
end
 
def kube_client
cluster&.kubeclient&.core_client
end
def migrate_to_3_script
return [] if !updating? || chart_above_v3?
 
Loading
Loading
Loading
Loading
@@ -52,6 +52,7 @@ class Cluster < ApplicationRecord
has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true
 
has_one :integration_prometheus, class_name: 'Clusters::Integrations::Prometheus', inverse_of: :cluster
has_one :integration_elastic_stack, class_name: 'Clusters::Integrations::ElasticStack', inverse_of: :cluster
 
def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateName
application = APPLICATIONS[name.to_s]
Loading
Loading
@@ -104,6 +105,7 @@ def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateNam
delegate :available?, to: :application_ingress, prefix: true, allow_nil: true
delegate :available?, to: :application_knative, prefix: true, allow_nil: true
delegate :available?, to: :application_elastic_stack, prefix: true, allow_nil: true
delegate :available?, to: :integration_elastic_stack, prefix: true, allow_nil: true
delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true
delegate :external_hostname, to: :application_ingress, prefix: true, allow_nil: true
 
Loading
Loading
@@ -284,6 +286,10 @@ def find_or_build_integration_prometheus
integration_prometheus || build_integration_prometheus
end
 
def find_or_build_integration_elastic_stack
integration_elastic_stack || build_integration_elastic_stack
end
def provider
if gcp?
provider_gcp
Loading
Loading
@@ -318,6 +324,22 @@ def kubeclient
platform_kubernetes.kubeclient if kubernetes?
end
 
def elastic_stack_adapter
application_elastic_stack || integration_elastic_stack
end
def elasticsearch_client
elastic_stack_adapter&.elasticsearch_client
end
def elastic_stack_available?
if application_elastic_stack_available? || integration_elastic_stack_available?
true
else
false
end
end
def kubernetes_namespace_for(environment, deployable: environment.last_deployable)
if deployable && environment.project_id != deployable.project_id
raise ArgumentError, 'environment.project_id must match deployable.project_id'
Loading
Loading
Loading
Loading
@@ -6,6 +6,8 @@ module ApplicationCore
extend ActiveSupport::Concern
 
included do
include ::Clusters::Concerns::KubernetesLogger
belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
 
validates :cluster, presence: true
Loading
Loading
@@ -79,24 +81,6 @@ def post_uninstall
# Override if your application needs any action after
# being uninstalled by Helm
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def log_exception(error, event)
logger.error({
exception: error.class.name,
status_code: error.error_code,
cluster_id: cluster&.id,
application_id: id,
class_name: self.class.name,
event: event,
message: error.message
})
Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
end
end
end
end
Loading
Loading
# frozen_string_literal: true
module Clusters
module Concerns
module ElasticsearchClient
include ::Gitlab::Utils::StrongMemoize
ELASTICSEARCH_PORT = 9200
ELASTICSEARCH_NAMESPACE = 'gitlab-managed-apps'
def elasticsearch_client(timeout: nil)
strong_memoize(:elasticsearch_client) do
kube_client = cluster&.kubeclient&.core_client
next unless kube_client
proxy_url = kube_client.proxy_url('service', service_name, ELASTICSEARCH_PORT, ELASTICSEARCH_NAMESPACE)
Elasticsearch::Client.new(url: proxy_url) do |faraday|
# ensures headers containing auth data are appended to original client options
faraday.headers.merge!(kube_client.headers)
# ensure TLS certs are properly verified
faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl]
faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store]
faraday.options.timeout = timeout unless timeout.nil?
end
rescue Kubeclient::HttpError => error
# If users have mistakenly set parameters or removed the depended clusters,
# `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
# We check for a nil client in downstream use and behaviour is equivalent to an empty state
log_exception(error, :failed_to_create_elasticsearch_client)
nil
end
end
end
end
end
# frozen_string_literal: true
module Clusters
module Concerns
module KubernetesLogger
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def log_exception(error, event)
logger.error(
{
exception: error.class.name,
status_code: error.error_code,
cluster_id: cluster&.id,
application_id: id,
class_name: self.class.name,
event: event,
message: error.message
}
)
Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
end
end
end
end
# frozen_string_literal: true
module Clusters
module Integrations
class ElasticStack < ApplicationRecord
include ::Clusters::Concerns::ElasticsearchClient
include ::Clusters::Concerns::KubernetesLogger
self.table_name = 'clusters_integration_elasticstack'
self.primary_key = :cluster_id
belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
validates :cluster, presence: true
validates :enabled, inclusion: { in: [true, false] }
def available?
enabled
end
def service_name
chart_above_v3? ? 'elastic-stack-elasticsearch-master' : 'elastic-stack-elasticsearch-client'
end
def chart_above_v2?
return true if chart_version.nil?
Gem::Version.new(chart_version) >= Gem::Version.new('2.0.0')
end
def chart_above_v3?
return true if chart_version.nil?
Gem::Version.new(chart_version) >= Gem::Version.new('3.0.0')
end
end
end
end
Loading
Loading
@@ -406,7 +406,7 @@ def auto_stop_in=(value)
end
 
def elastic_stack_available?
!!deployment_platform&.cluster&.application_elastic_stack_available?
!!deployment_platform&.cluster&.elastic_stack_available?
end
 
def rollout_status
Loading
Loading
Loading
Loading
@@ -76,7 +76,7 @@ def integrations_path
def gitlab_managed_apps_logs_path
return unless logs_project && can_read_cluster?
 
if cluster.application_elastic_stack&.available?
if cluster.elastic_stack_adapter&.available?
elasticsearch_project_logs_path(logs_project, cluster_id: cluster.id, format: :json)
else
k8s_project_logs_path(logs_project, cluster_id: cluster.id, format: :json)
Loading
Loading
Loading
Loading
@@ -28,6 +28,6 @@ class ClusterEntity < Grape::Entity
end
 
expose :enable_advanced_logs_querying do |cluster|
cluster.application_elastic_stack_available?
cluster.elastic_stack_available?
end
end
Loading
Loading
@@ -27,12 +27,15 @@ def execute
private
 
def integration
case params[:application_type]
when 'prometheus'
cluster.find_or_build_integration_prometheus
else
raise ArgumentError, "invalid application_type: #{params[:application_type]}"
end
@integration ||= \
case params[:application_type]
when 'prometheus'
cluster.find_or_build_integration_prometheus
when 'elastic_stack'
cluster.find_or_build_integration_elastic_stack
else
raise ArgumentError, "invalid application_type: #{params[:application_type]}"
end
end
 
def authorized?
Loading
Loading
Loading
Loading
@@ -24,7 +24,7 @@ def success_return_keys
end
 
def get_raw_pods(result)
client = cluster&.application_elastic_stack&.elasticsearch_client
client = cluster&.elasticsearch_client
return error(_('Unable to connect to Elasticsearch')) unless client
 
result[:raw_pods] = ::Gitlab::Elasticsearch::Logs::Pods.new(client).pods(namespace)
Loading
Loading
@@ -66,11 +66,9 @@ def check_cursor(result)
end
 
def pod_logs(result)
client = cluster&.application_elastic_stack&.elasticsearch_client
client = cluster&.elasticsearch_client
return error(_('Unable to connect to Elasticsearch')) unless client
 
chart_above_v2 = cluster.application_elastic_stack.chart_above_v2?
response = ::Gitlab::Elasticsearch::Logs::Lines.new(client).pod_logs(
namespace,
pod_name: result[:pod_name],
Loading
Loading
@@ -79,7 +77,7 @@ def pod_logs(result)
start_time: result[:start_time],
end_time: result[:end_time],
cursor: result[:cursor],
chart_above_v2: chart_above_v2
chart_above_v2: cluster.elastic_stack_adapter.chart_above_v2?
)
 
result.merge!(response)
Loading
Loading
.settings.expanded.border-0.m-0
%p
= s_('ClusterIntegration|Integrations enable you to integrate your cluster as part of your GitLab workflow.')
= s_('ClusterIntegration|Integrations allow you to use applications installed in your cluster as part of your GitLab workflow.')
= link_to _('Learn more'), help_page_path('user/clusters/integrations.md'), target: '_blank'
.settings-content#advanced-settings-section
.settings-content#integrations-settings-section
- if can?(current_user, :admin_cluster, @cluster)
.sub-section.form-group
= form_for @prometheus_integration, url: @cluster.integrations_path, as: :integration, method: :post, html: { class: 'js-cluster-integrations-form' } do |form|
= form.hidden_field :application_type
.form-group
= form_for @prometheus_integration, as: :integration, namespace: :prometheus, url: @cluster.integrations_path, method: :post, html: { class: 'js-cluster-integrations-form' } do |prometheus_form|
= prometheus_form.hidden_field :application_type
.form-group.gl-form-group
.gl-form-checkbox.custom-control.custom-checkbox
= form.check_box :enabled, { class: 'custom-control-input'}
= form.label :enabled, s_('ClusterIntegration|Enable Prometheus integration'), class: 'custom-control-label'
.gl-form-group
= prometheus_form.check_box :enabled, class: 'custom-control-input'
= prometheus_form.label :enabled, s_('ClusterIntegration|Enable Prometheus integration'), class: 'custom-control-label'
.form-text.text-gl-muted
- link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: help_page_path("user/clusters/integrations", anchor: "prometheus-cluster-integration") }
- link_end = '</a>'.html_safe
= html_escape(s_('ClusterIntegration|Before you enable this integration, follow the %{link_start}documented process%{link_end}.')) % { link_start: link_start, link_end: link_end }
= form.submit _('Save changes'), class: 'btn gl-button btn-success'
= s_('ClusterIntegration|Allows GitLab to query a specifically configured in-cluster Prometheus for metrics.')
= link_to _('More information.'), help_page_path("user/clusters/integrations", anchor: "prometheus-cluster-integration"), target: '_blank'
= prometheus_form.submit _('Save changes'), class: 'btn gl-button btn-success'
.sub-section.form-group
= form_for @elastic_stack_integration, as: :integration, namespace: :elastic_stack, url: @cluster.integrations_path, method: :post, html: { class: 'js-cluster-integrations-form' } do |elastic_stack_form|
= elastic_stack_form.hidden_field :application_type
.form-group.gl-form-group
.gl-form-checkbox.custom-control.custom-checkbox
= elastic_stack_form.check_box :enabled, class: 'custom-control-input'
= elastic_stack_form.label :enabled, s_('ClusterIntegration|Enable Elastic Stack integration'), class: 'custom-control-label'
.form-text.text-gl-muted
= s_('ClusterIntegration|Allows GitLab to query a specifically configured in-cluster Elasticsearch for pod logs.')
= link_to _('More information.'), help_page_path("user/clusters/integrations", anchor: "elastic-stack-cluster-integration"), target: '_blank'
= elastic_stack_form.submit _('Save changes'), class: 'btn gl-button btn-success'
---
title: Add Elastic Stack cluster integration
merge_request: 61077
author:
type: added
# frozen_string_literal: true
class CreateClustersIntegrationElasticstack < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
def change
create_table_with_constraints :clusters_integration_elasticstack, id: false do |t|
t.timestamps_with_timezone null: false
t.references :cluster, primary_key: true, default: nil, index: false, foreign_key: { on_delete: :cascade }
t.boolean :enabled, null: false, default: false
t.text :chart_version
t.text_limit :chart_version, 10
end
end
end
c4593c1638f937618ecf3ae94a409e550dce93cc190989f581fb0007e591696d
\ No newline at end of file
Loading
Loading
@@ -11715,6 +11715,15 @@ CREATE SEQUENCE clusters_id_seq
 
ALTER SEQUENCE clusters_id_seq OWNED BY clusters.id;
 
CREATE TABLE clusters_integration_elasticstack (
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
cluster_id bigint NOT NULL,
enabled boolean DEFAULT false NOT NULL,
chart_version text,
CONSTRAINT check_f8d671ce04 CHECK ((char_length(chart_version) <= 10))
);
CREATE TABLE clusters_integration_prometheus (
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
Loading
Loading
@@ -20734,6 +20743,9 @@ ALTER TABLE ONLY clusters_applications_prometheus
ALTER TABLE ONLY clusters_applications_runners
ADD CONSTRAINT clusters_applications_runners_pkey PRIMARY KEY (id);
 
ALTER TABLE ONLY clusters_integration_elasticstack
ADD CONSTRAINT clusters_integration_elasticstack_pkey PRIMARY KEY (cluster_id);
ALTER TABLE ONLY clusters_integration_prometheus
ADD CONSTRAINT clusters_integration_prometheus_pkey PRIMARY KEY (cluster_id);
 
Loading
Loading
@@ -26969,6 +26981,9 @@ ALTER TABLE ONLY boards_epic_board_positions
ALTER TABLE ONLY vulnerability_finding_links
ADD CONSTRAINT fk_rails_cbdfde27ce FOREIGN KEY (vulnerability_occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;
 
ALTER TABLE ONLY clusters_integration_elasticstack
ADD CONSTRAINT fk_rails_cc5ba8f658 FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE;
ALTER TABLE ONLY issues_self_managed_prometheus_alert_events
ADD CONSTRAINT fk_rails_cc5d88bbb0 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE;
 
Loading
Loading
@@ -10,7 +10,9 @@ GitLab provides several ways to integrate applications to your
Kubernetes cluster.
 
To enable cluster integrations, first add a Kubernetes cluster to a GitLab
[project](../project/clusters/add_remove_clusters.md) or [group](../group/clusters/index.md#group-level-kubernetes-clusters).
[project](../project/clusters/add_remove_clusters.md) or
[group](../group/clusters/index.md#group-level-kubernetes-clusters) or
[instance](../instance/clusters/index.md).
 
## Prometheus cluster integration
 
Loading
Loading
@@ -20,33 +22,33 @@ You can integrate your Kubernetes cluster with
[Prometheus](https://prometheus.io/) for monitoring key metrics of your
apps directly from the GitLab UI.
 
[Alerts](../../operations/metrics/alerts.md) are not currently
supported.
[Alerts](../../operations/metrics/alerts.md) can be configured the same way as
for [external Prometheus instances](../../operations/metrics/alerts.md#external-prometheus-instances).
 
Once enabled, you will see metrics from services available in the
Once enabled, you can see metrics from services available in the
[metrics library](../project/integrations/prometheus_library/index.md).
 
Prerequisites:
### Prometheus Prerequisites
 
To benefit from this integration, you must have Prometheus
installed in your cluster with the following requirements:
To use this integration:
 
1. Prometheus must be installed inside the `gitlab-managed-apps` namespace.
1. Prometheus must be installed in your cluster in the `gitlab-managed-apps` namespace.
1. The `Service` resource for Prometheus must be named `prometheus-prometheus-server`.
 
You can use the following commands to install Prometheus to meet the requirements for cluster integrations:
You can manage your Prometheus however you like, but as an example, you can set
it up using [Helm](https://helm.sh/) as follows:
 
```shell
# Create the require Kubernetes namespace
# Create the required Kubernetes namespace
kubectl create ns gitlab-managed-apps
 
# Download Helm chart values that is compatible with the requirements above.
# You should substitute the tag that corresponds to the GitLab version in the url
# You should substitute the tag that corresponds to the GitLab version in the URL
# - https://gitlab.com/gitlab-org/gitlab/-/raw/<tag>/vendor/prometheus/values.yaml
#
wget https://gitlab.com/gitlab-org/gitlab/-/raw/v13.9.0-ee/vendor/prometheus/values.yaml
 
# Add the Prometheus community helm repo
# Add the Prometheus community Helm chart repository
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 
# Install Prometheus
Loading
Loading
@@ -65,6 +67,65 @@ To enable the Prometheus integration for your cluster:
**Operations > Kubernetes**.
- For a [group-level cluster](../group/clusters/index.md), navigate to your group's
**Kubernetes** page.
- For an [instance-level cluster](../instance/clusters/index.md), navigate to your instance's
**Kubernetes** page.
1. Select the **Integrations** tab.
1. Check the **Enable Prometheus integration** checkbox.
1. Click **Save changes**.
1. Go to the **Health** tab to see your cluster's metrics.
## Elastic Stack cluster integration
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61077) in GitLab 13.12.
You can integrate your cluster with [Elastic
Stack](https://www.elastic.co/elastic-stack) to index and [query your pod
logs](../project/clusters/kubernetes_pod_logs.md).
### Elastic Stack Prerequisites
To use this integration:
1. Elasticsearch 7.x or must be installed in your cluster in the
`gitlab-managed-apps` namespace.
1. The `Service` resource must be called `elastic-stack-elasticsearch-master`
and expose the Elasticsearch API on port `9200`.
1. The logs are expected to be [Filebeat container logs](https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-input-container.html)
following the [7.x log structure](https://www.elastic.co/guide/en/beats/filebeat/7.x/exported-fields-log.html)
and include [Kubernetes metadata](https://www.elastic.co/guide/en/beats/filebeat/7.x/add-kubernetes-metadata.html).
You can manage your Elastic Stack however you like, but as an example, you can
use [this Elastic Stack chart](https://gitlab.com/gitlab-org/charts/elastic-stack) to get up and
running:
```shell
# Create the required Kubernetes namespace
kubectl create namespace gitlab-managed-apps
# Download Helm chart values that is compatible with the requirements above.
# You should substitute the tag that corresponds to the GitLab version in the URL
# - https://gitlab.com/gitlab-org/gitlab/-/raw/<tag>/vendor/elastic_stack/values.yaml
#
wget https://gitlab.com/gitlab-org/gitlab/-/raw/v13.9.0-ee/vendor/elastic_stack/values.yaml
# Add the GitLab Helm chart repository
helm repo add gitlab https://charts.gitlab.io
# Install Elastic Stack
helm install prometheus gitlab/elastic-stack -n gitlab-managed-apps --values values.yaml
```
### Enable Elastic Stack integration for your cluster
To enable the Elastic Stack integration for your cluster:
1. Go to the cluster's page:
- For a [project-level cluster](../project/clusters/index.md), navigate to your project's
**Operations > Kubernetes**.
- For a [group-level cluster](../group/clusters/index.md), navigate to your group's
**Kubernetes** page.
- For an [instance-level cluster](../instance/clusters/index.md), navigate to your instance's
**Kubernetes** page.
1. Select the **Integrations** tab.
1. Check the **Enable Prometheus integration** checkbox.
1. Click **Save changes**.
Loading
Loading
Loading
Loading
@@ -50,17 +50,17 @@ def execute(totals_only: false)
end
 
def elasticsearch_client
@elasticsearch_client ||= application_elastic_stack&.elasticsearch_client(timeout: @options[:timeout])
@elasticsearch_client ||= elastic_stack_adapter&.elasticsearch_client(timeout: @options[:timeout])
end
 
private
 
def application_elastic_stack
@application_elastic_stack ||= @cluster&.application_elastic_stack
def elastic_stack_adapter
@elastic_stack_adapter ||= @cluster&.elastic_stack_adapter
end
 
def chart_above_v3?
application_elastic_stack.chart_above_v3?
elastic_stack_adapter.chart_above_v3?
end
 
def body
Loading
Loading
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment