Skip to content
Snippets Groups Projects
Commit 09f5face authored by Thong Kuah's avatar Thong Kuah :speech_balloon:
Browse files

Merge branch 'add-elasticstack-cluster-integration' into 'master'

Add Elastic Stack cluster integration

See merge request gitlab-org/gitlab!61077
parents aaeced81 d36ebb34
No related branches found
No related tags found
No related merge requests found
Showing
with 295 additions and 93 deletions
Loading
Loading
@@ -62,6 +62,7 @@ def cluster_status
def show
if params[:tab] == 'integrations'
@prometheus_integration = Clusters::IntegrationPresenter.new(@cluster.find_or_build_integration_prometheus)
@elastic_stack_integration = Clusters::IntegrationPresenter.new(@cluster.find_or_build_integration_elastic_stack)
end
end
 
Loading
Loading
Loading
Loading
@@ -24,7 +24,7 @@ def clusterable
end
 
def cluster_integration_params
params.require(:integration).permit(:application_type, :enabled)
params.permit(integration: [:enabled, :application_type]).require(:integration)
end
 
def cluster
Loading
Loading
Loading
Loading
@@ -3,9 +3,9 @@
module Clusters
module Applications
class ElasticStack < ApplicationRecord
VERSION = '3.0.0'
include ::Clusters::Concerns::ElasticsearchClient
 
ELASTICSEARCH_PORT = 9200
VERSION = '3.0.0'
 
self.table_name = 'clusters_applications_elastic_stacks'
 
Loading
Loading
@@ -13,10 +13,23 @@ class ElasticStack < ApplicationRecord
include ::Clusters::Concerns::ApplicationStatus
include ::Clusters::Concerns::ApplicationVersion
include ::Clusters::Concerns::ApplicationData
include ::Gitlab::Utils::StrongMemoize
 
default_value_for :version, VERSION
 
after_destroy do
cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil)
end
state_machine :status do
after_transition any => [:installed] do |application|
application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: true, chart_version: application.version)
end
after_transition any => [:uninstalled] do |application|
application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil)
end
end
def chart
'elastic-stack/elastic-stack'
end
Loading
Loading
@@ -51,31 +64,6 @@ def files
super.merge('wait-for-elasticsearch.sh': File.read("#{Rails.root}/vendor/elastic_stack/wait-for-elasticsearch.sh"))
end
 
def elasticsearch_client(timeout: nil)
strong_memoize(:elasticsearch_client) do
next unless kube_client
proxy_url = kube_client.proxy_url('service', service_name, ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE)
Elasticsearch::Client.new(url: proxy_url) do |faraday|
# ensures headers containing auth data are appended to original client options
faraday.headers.merge!(kube_client.headers)
# ensure TLS certs are properly verified
faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl]
faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store]
faraday.options.timeout = timeout unless timeout.nil?
end
rescue Kubeclient::HttpError => error
# If users have mistakenly set parameters or removed the depended clusters,
# `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
# We check for a nil client in downstream use and behaviour is equivalent to an empty state
log_exception(error, :failed_to_create_elasticsearch_client)
nil
end
end
def chart_above_v2?
Gem::Version.new(version) >= Gem::Version.new('2.0.0')
end
Loading
Loading
@@ -106,10 +94,6 @@ def post_delete_script
]
end
 
def kube_client
cluster&.kubeclient&.core_client
end
def migrate_to_3_script
return [] if !updating? || chart_above_v3?
 
Loading
Loading
Loading
Loading
@@ -52,6 +52,7 @@ class Cluster < ApplicationRecord
has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true
 
has_one :integration_prometheus, class_name: 'Clusters::Integrations::Prometheus', inverse_of: :cluster
has_one :integration_elastic_stack, class_name: 'Clusters::Integrations::ElasticStack', inverse_of: :cluster
 
def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateName
application = APPLICATIONS[name.to_s]
Loading
Loading
@@ -104,6 +105,7 @@ def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateNam
delegate :available?, to: :application_ingress, prefix: true, allow_nil: true
delegate :available?, to: :application_knative, prefix: true, allow_nil: true
delegate :available?, to: :application_elastic_stack, prefix: true, allow_nil: true
delegate :available?, to: :integration_elastic_stack, prefix: true, allow_nil: true
delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true
delegate :external_hostname, to: :application_ingress, prefix: true, allow_nil: true
 
Loading
Loading
@@ -284,6 +286,10 @@ def find_or_build_integration_prometheus
integration_prometheus || build_integration_prometheus
end
 
def find_or_build_integration_elastic_stack
integration_elastic_stack || build_integration_elastic_stack
end
def provider
if gcp?
provider_gcp
Loading
Loading
@@ -318,6 +324,22 @@ def kubeclient
platform_kubernetes.kubeclient if kubernetes?
end
 
def elastic_stack_adapter
application_elastic_stack || integration_elastic_stack
end
def elasticsearch_client
elastic_stack_adapter&.elasticsearch_client
end
def elastic_stack_available?
if application_elastic_stack_available? || integration_elastic_stack_available?
true
else
false
end
end
def kubernetes_namespace_for(environment, deployable: environment.last_deployable)
if deployable && environment.project_id != deployable.project_id
raise ArgumentError, 'environment.project_id must match deployable.project_id'
Loading
Loading
Loading
Loading
@@ -6,6 +6,8 @@ module ApplicationCore
extend ActiveSupport::Concern
 
included do
include ::Clusters::Concerns::KubernetesLogger
belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
 
validates :cluster, presence: true
Loading
Loading
@@ -79,24 +81,6 @@ def post_uninstall
# Override if your application needs any action after
# being uninstalled by Helm
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def log_exception(error, event)
logger.error({
exception: error.class.name,
status_code: error.error_code,
cluster_id: cluster&.id,
application_id: id,
class_name: self.class.name,
event: event,
message: error.message
})
Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
end
end
end
end
Loading
Loading
# frozen_string_literal: true
module Clusters
module Concerns
module ElasticsearchClient
include ::Gitlab::Utils::StrongMemoize
ELASTICSEARCH_PORT = 9200
ELASTICSEARCH_NAMESPACE = 'gitlab-managed-apps'
def elasticsearch_client(timeout: nil)
strong_memoize(:elasticsearch_client) do
kube_client = cluster&.kubeclient&.core_client
next unless kube_client
proxy_url = kube_client.proxy_url('service', service_name, ELASTICSEARCH_PORT, ELASTICSEARCH_NAMESPACE)
Elasticsearch::Client.new(url: proxy_url) do |faraday|
# ensures headers containing auth data are appended to original client options
faraday.headers.merge!(kube_client.headers)
# ensure TLS certs are properly verified
faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl]
faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store]
faraday.options.timeout = timeout unless timeout.nil?
end
rescue Kubeclient::HttpError => error
# If users have mistakenly set parameters or removed the depended clusters,
# `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
# We check for a nil client in downstream use and behaviour is equivalent to an empty state
log_exception(error, :failed_to_create_elasticsearch_client)
nil
end
end
end
end
end
# frozen_string_literal: true
module Clusters
module Concerns
module KubernetesLogger
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def log_exception(error, event)
logger.error(
{
exception: error.class.name,
status_code: error.error_code,
cluster_id: cluster&.id,
application_id: id,
class_name: self.class.name,
event: event,
message: error.message
}
)
Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
end
end
end
end
# frozen_string_literal: true
module Clusters
module Integrations
class ElasticStack < ApplicationRecord
include ::Clusters::Concerns::ElasticsearchClient
include ::Clusters::Concerns::KubernetesLogger
self.table_name = 'clusters_integration_elasticstack'
self.primary_key = :cluster_id
belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
validates :cluster, presence: true
validates :enabled, inclusion: { in: [true, false] }
def available?
enabled
end
def service_name
chart_above_v3? ? 'elastic-stack-elasticsearch-master' : 'elastic-stack-elasticsearch-client'
end
def chart_above_v2?
return true if chart_version.nil?
Gem::Version.new(chart_version) >= Gem::Version.new('2.0.0')
end
def chart_above_v3?
return true if chart_version.nil?
Gem::Version.new(chart_version) >= Gem::Version.new('3.0.0')
end
end
end
end
Loading
Loading
@@ -406,7 +406,7 @@ def auto_stop_in=(value)
end
 
def elastic_stack_available?
!!deployment_platform&.cluster&.application_elastic_stack_available?
!!deployment_platform&.cluster&.elastic_stack_available?
end
 
def rollout_status
Loading
Loading
Loading
Loading
@@ -76,7 +76,7 @@ def integrations_path
def gitlab_managed_apps_logs_path
return unless logs_project && can_read_cluster?
 
if cluster.application_elastic_stack&.available?
if cluster.elastic_stack_adapter&.available?
elasticsearch_project_logs_path(logs_project, cluster_id: cluster.id, format: :json)
else
k8s_project_logs_path(logs_project, cluster_id: cluster.id, format: :json)
Loading
Loading
Loading
Loading
@@ -28,6 +28,6 @@ class ClusterEntity < Grape::Entity
end
 
expose :enable_advanced_logs_querying do |cluster|
cluster.application_elastic_stack_available?
cluster.elastic_stack_available?
end
end
Loading
Loading
@@ -27,12 +27,15 @@ def execute
private
 
def integration
case params[:application_type]
when 'prometheus'
cluster.find_or_build_integration_prometheus
else
raise ArgumentError, "invalid application_type: #{params[:application_type]}"
end
@integration ||= \
case params[:application_type]
when 'prometheus'
cluster.find_or_build_integration_prometheus
when 'elastic_stack'
cluster.find_or_build_integration_elastic_stack
else
raise ArgumentError, "invalid application_type: #{params[:application_type]}"
end
end
 
def authorized?
Loading
Loading
Loading
Loading
@@ -24,7 +24,7 @@ def success_return_keys
end
 
def get_raw_pods(result)
client = cluster&.application_elastic_stack&.elasticsearch_client
client = cluster&.elasticsearch_client
return error(_('Unable to connect to Elasticsearch')) unless client
 
result[:raw_pods] = ::Gitlab::Elasticsearch::Logs::Pods.new(client).pods(namespace)
Loading
Loading
@@ -66,11 +66,9 @@ def check_cursor(result)
end
 
def pod_logs(result)
client = cluster&.application_elastic_stack&.elasticsearch_client
client = cluster&.elasticsearch_client
return error(_('Unable to connect to Elasticsearch')) unless client
 
chart_above_v2 = cluster.application_elastic_stack.chart_above_v2?
response = ::Gitlab::Elasticsearch::Logs::Lines.new(client).pod_logs(
namespace,
pod_name: result[:pod_name],
Loading
Loading
@@ -79,7 +77,7 @@ def pod_logs(result)
start_time: result[:start_time],
end_time: result[:end_time],
cursor: result[:cursor],
chart_above_v2: chart_above_v2
chart_above_v2: cluster.elastic_stack_adapter.chart_above_v2?
)
 
result.merge!(response)
Loading
Loading
.settings.expanded.border-0.m-0
%p
= s_('ClusterIntegration|Integrations enable you to integrate your cluster as part of your GitLab workflow.')
= s_('ClusterIntegration|Integrations allow you to use applications installed in your cluster as part of your GitLab workflow.')
= link_to _('Learn more'), help_page_path('user/clusters/integrations.md'), target: '_blank'
.settings-content#advanced-settings-section
.settings-content#integrations-settings-section
- if can?(current_user, :admin_cluster, @cluster)
.sub-section.form-group
= form_for @prometheus_integration, url: @cluster.integrations_path, as: :integration, method: :post, html: { class: 'js-cluster-integrations-form' } do |form|
= form.hidden_field :application_type
.form-group
= form_for @prometheus_integration, as: :integration, namespace: :prometheus, url: @cluster.integrations_path, method: :post, html: { class: 'js-cluster-integrations-form' } do |prometheus_form|
= prometheus_form.hidden_field :application_type
.form-group.gl-form-group
.gl-form-checkbox.custom-control.custom-checkbox
= form.check_box :enabled, { class: 'custom-control-input'}
= form.label :enabled, s_('ClusterIntegration|Enable Prometheus integration'), class: 'custom-control-label'
.gl-form-group
= prometheus_form.check_box :enabled, class: 'custom-control-input'
= prometheus_form.label :enabled, s_('ClusterIntegration|Enable Prometheus integration'), class: 'custom-control-label'
.form-text.text-gl-muted
- link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: help_page_path("user/clusters/integrations", anchor: "prometheus-cluster-integration") }
- link_end = '</a>'.html_safe
= html_escape(s_('ClusterIntegration|Before you enable this integration, follow the %{link_start}documented process%{link_end}.')) % { link_start: link_start, link_end: link_end }
= form.submit _('Save changes'), class: 'btn gl-button btn-success'
= s_('ClusterIntegration|Allows GitLab to query a specifically configured in-cluster Prometheus for metrics.')
= link_to _('More information.'), help_page_path("user/clusters/integrations", anchor: "prometheus-cluster-integration"), target: '_blank'
= prometheus_form.submit _('Save changes'), class: 'btn gl-button btn-success'
.sub-section.form-group
= form_for @elastic_stack_integration, as: :integration, namespace: :elastic_stack, url: @cluster.integrations_path, method: :post, html: { class: 'js-cluster-integrations-form' } do |elastic_stack_form|
= elastic_stack_form.hidden_field :application_type
.form-group.gl-form-group
.gl-form-checkbox.custom-control.custom-checkbox
= elastic_stack_form.check_box :enabled, class: 'custom-control-input'
= elastic_stack_form.label :enabled, s_('ClusterIntegration|Enable Elastic Stack integration'), class: 'custom-control-label'
.form-text.text-gl-muted
= s_('ClusterIntegration|Allows GitLab to query a specifically configured in-cluster Elasticsearch for pod logs.')
= link_to _('More information.'), help_page_path("user/clusters/integrations", anchor: "elastic-stack-cluster-integration"), target: '_blank'
= elastic_stack_form.submit _('Save changes'), class: 'btn gl-button btn-success'
---
title: Add Elastic Stack cluster integration
merge_request: 61077
author:
type: added
# frozen_string_literal: true
class CreateClustersIntegrationElasticstack < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
def change
create_table_with_constraints :clusters_integration_elasticstack, id: false do |t|
t.timestamps_with_timezone null: false
t.references :cluster, primary_key: true, default: nil, index: false, foreign_key: { on_delete: :cascade }
t.boolean :enabled, null: false, default: false
t.text :chart_version
t.text_limit :chart_version, 10
end
end
end
c4593c1638f937618ecf3ae94a409e550dce93cc190989f581fb0007e591696d
\ No newline at end of file
Loading
Loading
@@ -11719,6 +11719,15 @@ CREATE SEQUENCE clusters_id_seq
 
ALTER SEQUENCE clusters_id_seq OWNED BY clusters.id;
 
CREATE TABLE clusters_integration_elasticstack (
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
cluster_id bigint NOT NULL,
enabled boolean DEFAULT false NOT NULL,
chart_version text,
CONSTRAINT check_f8d671ce04 CHECK ((char_length(chart_version) <= 10))
);
CREATE TABLE clusters_integration_prometheus (
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
Loading
Loading
@@ -20796,6 +20805,9 @@ ALTER TABLE ONLY clusters_applications_prometheus
ALTER TABLE ONLY clusters_applications_runners
ADD CONSTRAINT clusters_applications_runners_pkey PRIMARY KEY (id);
 
ALTER TABLE ONLY clusters_integration_elasticstack
ADD CONSTRAINT clusters_integration_elasticstack_pkey PRIMARY KEY (cluster_id);
ALTER TABLE ONLY clusters_integration_prometheus
ADD CONSTRAINT clusters_integration_prometheus_pkey PRIMARY KEY (cluster_id);
 
Loading
Loading
@@ -27064,6 +27076,9 @@ ALTER TABLE ONLY boards_epic_board_positions
ALTER TABLE ONLY vulnerability_finding_links
ADD CONSTRAINT fk_rails_cbdfde27ce FOREIGN KEY (vulnerability_occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;
 
ALTER TABLE ONLY clusters_integration_elasticstack
ADD CONSTRAINT fk_rails_cc5ba8f658 FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE;
ALTER TABLE ONLY issues_self_managed_prometheus_alert_events
ADD CONSTRAINT fk_rails_cc5d88bbb0 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE;
 
Loading
Loading
@@ -10,7 +10,9 @@ GitLab provides several ways to integrate applications to your
Kubernetes cluster.
 
To enable cluster integrations, first add a Kubernetes cluster to a GitLab
[project](../project/clusters/add_remove_clusters.md) or [group](../group/clusters/index.md#group-level-kubernetes-clusters).
[project](../project/clusters/add_remove_clusters.md) or
[group](../group/clusters/index.md#group-level-kubernetes-clusters) or
[instance](../instance/clusters/index.md).
 
## Prometheus cluster integration
 
Loading
Loading
@@ -20,33 +22,33 @@ You can integrate your Kubernetes cluster with
[Prometheus](https://prometheus.io/) for monitoring key metrics of your
apps directly from the GitLab UI.
 
[Alerts](../../operations/metrics/alerts.md) are not currently
supported.
[Alerts](../../operations/metrics/alerts.md) can be configured the same way as
for [external Prometheus instances](../../operations/metrics/alerts.md#external-prometheus-instances).
 
Once enabled, you will see metrics from services available in the
Once enabled, you can see metrics from services available in the
[metrics library](../project/integrations/prometheus_library/index.md).
 
Prerequisites:
### Prometheus Prerequisites
 
To benefit from this integration, you must have Prometheus
installed in your cluster with the following requirements:
To use this integration:
 
1. Prometheus must be installed inside the `gitlab-managed-apps` namespace.
1. Prometheus must be installed in your cluster in the `gitlab-managed-apps` namespace.
1. The `Service` resource for Prometheus must be named `prometheus-prometheus-server`.
 
You can use the following commands to install Prometheus to meet the requirements for cluster integrations:
You can manage your Prometheus however you like, but as an example, you can set
it up using [Helm](https://helm.sh/) as follows:
 
```shell
# Create the require Kubernetes namespace
# Create the required Kubernetes namespace
kubectl create ns gitlab-managed-apps
 
# Download Helm chart values that is compatible with the requirements above.
# You should substitute the tag that corresponds to the GitLab version in the url
# You should substitute the tag that corresponds to the GitLab version in the URL
# - https://gitlab.com/gitlab-org/gitlab/-/raw/<tag>/vendor/prometheus/values.yaml
#
wget https://gitlab.com/gitlab-org/gitlab/-/raw/v13.9.0-ee/vendor/prometheus/values.yaml
 
# Add the Prometheus community helm repo
# Add the Prometheus community Helm chart repository
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 
# Install Prometheus
Loading
Loading
@@ -65,6 +67,65 @@ To enable the Prometheus integration for your cluster:
**Operations > Kubernetes**.
- For a [group-level cluster](../group/clusters/index.md), navigate to your group's
**Kubernetes** page.
- For an [instance-level cluster](../instance/clusters/index.md), navigate to your instance's
**Kubernetes** page.
1. Select the **Integrations** tab.
1. Check the **Enable Prometheus integration** checkbox.
1. Click **Save changes**.
1. Go to the **Health** tab to see your cluster's metrics.
## Elastic Stack cluster integration
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61077) in GitLab 13.12.
You can integrate your cluster with [Elastic
Stack](https://www.elastic.co/elastic-stack) to index and [query your pod
logs](../project/clusters/kubernetes_pod_logs.md).
### Elastic Stack Prerequisites
To use this integration:
1. Elasticsearch 7.x or must be installed in your cluster in the
`gitlab-managed-apps` namespace.
1. The `Service` resource must be called `elastic-stack-elasticsearch-master`
and expose the Elasticsearch API on port `9200`.
1. The logs are expected to be [Filebeat container logs](https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-input-container.html)
following the [7.x log structure](https://www.elastic.co/guide/en/beats/filebeat/7.x/exported-fields-log.html)
and include [Kubernetes metadata](https://www.elastic.co/guide/en/beats/filebeat/7.x/add-kubernetes-metadata.html).
You can manage your Elastic Stack however you like, but as an example, you can
use [this Elastic Stack chart](https://gitlab.com/gitlab-org/charts/elastic-stack) to get up and
running:
```shell
# Create the required Kubernetes namespace
kubectl create namespace gitlab-managed-apps
# Download Helm chart values that is compatible with the requirements above.
# You should substitute the tag that corresponds to the GitLab version in the URL
# - https://gitlab.com/gitlab-org/gitlab/-/raw/<tag>/vendor/elastic_stack/values.yaml
#
wget https://gitlab.com/gitlab-org/gitlab/-/raw/v13.9.0-ee/vendor/elastic_stack/values.yaml
# Add the GitLab Helm chart repository
helm repo add gitlab https://charts.gitlab.io
# Install Elastic Stack
helm install prometheus gitlab/elastic-stack -n gitlab-managed-apps --values values.yaml
```
### Enable Elastic Stack integration for your cluster
To enable the Elastic Stack integration for your cluster:
1. Go to the cluster's page:
- For a [project-level cluster](../project/clusters/index.md), navigate to your project's
**Operations > Kubernetes**.
- For a [group-level cluster](../group/clusters/index.md), navigate to your group's
**Kubernetes** page.
- For an [instance-level cluster](../instance/clusters/index.md), navigate to your instance's
**Kubernetes** page.
1. Select the **Integrations** tab.
1. Check the **Enable Prometheus integration** checkbox.
1. Click **Save changes**.
Loading
Loading
Loading
Loading
@@ -50,17 +50,17 @@ def execute(totals_only: false)
end
 
def elasticsearch_client
@elasticsearch_client ||= application_elastic_stack&.elasticsearch_client(timeout: @options[:timeout])
@elasticsearch_client ||= elastic_stack_adapter&.elasticsearch_client(timeout: @options[:timeout])
end
 
private
 
def application_elastic_stack
@application_elastic_stack ||= @cluster&.application_elastic_stack
def elastic_stack_adapter
@elastic_stack_adapter ||= @cluster&.elastic_stack_adapter
end
 
def chart_above_v3?
application_elastic_stack.chart_above_v3?
elastic_stack_adapter.chart_above_v3?
end
 
def body
Loading
Loading
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment