Add support for Jupyter in GitLab via Kubernetes

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

app/assets/javascripts/clusters/clusters_bundle.js

@@ -31,6 +31,7 @@ export default class Clusters {
       installHelmPath,
       installIngressPath,
       installRunnerPath,
+      installJupyterPath,
       installPrometheusPath,
       managePrometheusPath,
       clusterStatus,
@@ -51,6 +52,7 @@ export default class Clusters {
       installIngressEndpoint: installIngressPath,
       installRunnerEndpoint: installRunnerPath,
       installPrometheusEndpoint: installPrometheusPath,
+      installJupyterEndpoint: installJupyterPath,
     });
     this.installApplication = this.installApplication.bind(this);

app/assets/javascripts/clusters/components/applications.vue

@@ -37,6 +37,11 @@ export default {
       default: '',
     },
   },
+  data() {
+    return {
+      jupyterSuggestHostnameValue: '',
+    };
+  },
   computed: {
     generalApplicationDescription() {
       return sprintf(
@@ -121,6 +126,20 @@ export default {
         false,
       );
     },
+    jupyterInstalled() {
+      return this.applications.jupyter.status === APPLICATION_INSTALLED;
+    },
+    jupyterHostname() {
+      return this.applications.jupyter.hostname;
+    },
+    jupyterSuggestHostname() {
+      return `jupyter.${this.applications.ingress.externalIp}.xip.io`;
+    },
+  },
+  watch: {
+    jupyterSuggestHostname() {
+      this.jupyterSuggestHostnameValue = this.jupyterSuggestHostname;
+    },
   },
 };
 </script>
@@ -278,11 +297,89 @@ export default {
               applications to production.`) }}
           </div>
         </application-row>
+        <application-row
+          id="jupyter"
+          :title="applications.jupyter.title"
+          title-link="https://jupyterhub.readthedocs.io/en/stable/"
+          :status="applications.jupyter.status"
+          :status-reason="applications.jupyter.statusReason"
+          :request-status="applications.jupyter.requestStatus"
+          :request-reason="applications.jupyter.requestReason"
+        >
+          <div slot="description">
+            <p>
+              {{ s__(`ClusterIntegration|JupyterHub, a multi-user Hub, spawns,
+                manages, and proxies multiple instances of the single-user
+                Jupyter notebook server. JupyterHub can be used to serve
+                notebooks to a class of students, a corporate data science group,
+                or a scientific research group.`) }}
+            </p>
+            <template v-if="jupyterInstalled">
+              <div class="form-group">
+                <label for="jupyter-hostname">
+                  {{ s__('ClusterIntegration|Jupyter Hostname') }}
+                </label>
+                <div
+                  v-if="jupyterHostname"
+                  class="input-group"
+                >
+                  <input
+                    type="text"
+                    id="jupyter-hostname"
+                    class="form-control js-hostname"
+                    :value="jupyterHostname"
+                    readonly
+                  />
+                  <span class="input-group-btn">
+                    <clipboard-button
+                      :text="jupyterHostname"
+                      :title="s__('ClusterIntegration|Copy Jupyter Hostname to clipboard')"
+                      class="js-clipboard-btn"
+                    />
+                  </span>
+                </div>
+              </div>
+            </template>
+            <template v-else-if="ingressInstalled">
+              <div class="form-group">
+                <label for="jupyter-hostname">
+                  {{ s__('ClusterIntegration|Jupyter Hostname') }}
+                </label>
+                <div class="input-group">
+                  <input
+                    type="text"
+                    id="jupyter-hostname"
+                    class="form-control js-hostname"
+                    v-model="jupyterSuggestHostnameValue"
+                  />
+                  <span class="input-group-btn">
+                    <clipboard-button
+                      :text="jupyterHostname"
+                      :title="s__('ClusterIntegration|Copy Jupyter Hostname to clipboard')"
+                      class="js-clipboard-btn"
+                    />
+                  </span>
+                </div>
+              </div>
+              <p>
+                {{ s__(`ClusterIntegration|Replace this with your own hostname if you want.
+                If you do so, point hostname to Ingress IP Address from above.`) }}
+                <a
+                  :href="ingressDnsHelpPath"
+                  target="_blank"
+                  rel="noopener noreferrer"
+                >
+                  {{ __('More information') }}
+                </a>
+              </p>
+            </template>
+          </div>
+        </application-row>
         <!--
           NOTE: Don't forget to update `clusters.scss`
           min-height for this block and uncomment `application_spec` tests
         -->
-        <!-- Add GitLab Runner row, all other plumbing is complete -->
+        <!-- Add Jupyter row, all other plumbing is complete -->
       </div>
     </div>
   </section>

app/assets/javascripts/clusters/constants.js

@@ -11,3 +11,4 @@ export const REQUEST_LOADING = 'request-loading';
 export const REQUEST_SUCCESS = 'request-success';
 export const REQUEST_FAILURE = 'request-failure';
 export const INGRESS = 'ingress';
+export const JUPYTER = 'jupyter';

app/assets/javascripts/clusters/services/clusters_service.js

 import axios from '../../lib/utils/axios_utils';
+import { JUPYTER } from '../constants';
 export default class ClusterService {
   constructor(options = {}) {
@@ -8,6 +9,7 @@ export default class ClusterService {
       ingress: this.options.installIngressEndpoint,
       runner: this.options.installRunnerEndpoint,
       prometheus: this.options.installPrometheusEndpoint,
+      jupyter: this.options.installJupyterEndpoint,
     };
   }
@@ -16,7 +18,13 @@ export default class ClusterService {
   }
   installApplication(appId) {
-    return axios.post(this.appInstallEndpointMap[appId]);
+    const data = {};
+
+    if (appId === JUPYTER) {
+      data.hostname = document.getElementById('jupyter-hostname').value;
+    }
+
+    return axios.post(this.appInstallEndpointMap[appId], data);
   }
   static updateCluster(endpoint, data) {

app/assets/javascripts/clusters/stores/clusters_store.js

 import { s__ } from '../../locale';
-import { INGRESS } from '../constants';
+import { INGRESS, JUPYTER } from '../constants';
 export default class ClusterStore {
   constructor() {
@@ -38,6 +38,14 @@ export default class ClusterStore {
           requestStatus: null,
           requestReason: null,
         },
+        jupyter: {
+          title: s__('ClusterIntegration|JupyterHub'),
+          status: null,
+          statusReason: null,
+          requestStatus: null,
+          requestReason: null,
+          hostname: null,
+        },
       },
     };
   }
@@ -83,6 +91,8 @@ export default class ClusterStore {
       if (appId === INGRESS) {
         this.state.applications.ingress.externalIp = serverAppEntry.external_ip;
+      } else if (appId === JUPYTER) {
+        this.state.applications.jupyter.hostname = serverAppEntry.hostname;
       }
     });
   }

app/assets/stylesheets/pages/clusters.scss

@@ -6,7 +6,7 @@
 .cluster-applications-table {
   // Wait for the Vue to kick-in and render the applications block
-  min-height: 400px;
+  min-height: 500px;
 }
 .clusters-dropdown-menu {

app/controllers/projects/clusters/applications_controller.rb

@@ -6,6 +6,7 @@ class Projects::Clusters::ApplicationsController < Projects::ApplicationControll
   def create
     application = @application_class.find_or_create_by!(cluster: @cluster)
+    application.update(hostname: params[:hostname]) if application.respond_to?(:hostname)
     Clusters::Applications::ScheduleInstallationService.new(project, current_user).execute(application)

app/models/clusters/applications/jupyter.rb

@@ -12,17 +12,39 @@ module Clusters
       default_value_for :version, VERSION
       def chart
-        # TODO: publish jupyterhub charts that we can use for our installation
-        # and provide path to it here.
+        "#{name}/jupyterhub"
+      end
+
+      def repository
+        'https://jupyterhub.github.io/helm-chart/'
+      end
+
+      def values
+        content_values.to_yaml
       end
       def install_command
         Gitlab::Kubernetes::Helm::InstallCommand.new(
           name,
           chart: chart,
-          values: values
+          values: values,
+          repository: repository
         )
       end
+
+      private
+
+      def specification
+        {
+          "ingress" => { "hosts" => [hostname] },
+          "hub" => { "cookieSecret" => SecureRandom.hex(32) },
+          "proxy" => { "secretToken" => SecureRandom.hex(32) }
+        }
+      end
+
+      def content_values
+        YAML.load_file(chart_values_file).deep_merge!(specification)
+      end
     end
   end
 end

app/models/clusters/cluster.rb

@@ -27,6 +27,7 @@ module Clusters
     has_one :application_ingress, class_name: 'Clusters::Applications::Ingress'
     has_one :application_prometheus, class_name: 'Clusters::Applications::Prometheus'
     has_one :application_runner, class_name: 'Clusters::Applications::Runner'
+    has_one :application_jupyter, class_name: 'Clusters::Applications::Jupyter'
     accepts_nested_attributes_for :provider_gcp, update_only: true
     accepts_nested_attributes_for :platform_kubernetes, update_only: true
@@ -75,7 +76,8 @@ module Clusters
         application_helm || build_application_helm,
         application_ingress || build_application_ingress,
         application_prometheus || build_application_prometheus,
-        application_runner || build_application_runner
+        application_runner || build_application_runner,
+        application_jupyter || build_application_jupyter
       ]
     end

app/serializers/cluster_application_entity.rb

@@ -3,4 +3,5 @@ class ClusterApplicationEntity < Grape::Entity
   expose :status_name, as: :status
   expose :status_reason
   expose :external_ip, if: -> (e, _) { e.respond_to?(:external_ip) }
+  expose :hostname, if: -> (e, _) { e.respond_to?(:hostname) }
 end

app/services/clusters/applications/install_service.rb

@@ -12,8 +12,8 @@ module Clusters
             ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
         rescue Kubeclient::HttpError => ke
           app.make_errored!("Kubernetes error: #{ke.message}")
-        rescue StandardError
-          app.make_errored!("Can't start installation process")
+        rescue StandardError => e
+          app.make_errored!("Can't start installation process. #{e.message}")
         end
       end
     end

app/views/projects/clusters/show.html.haml

@@ -11,6 +11,7 @@
   install_ingress_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :ingress),
   install_prometheus_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :prometheus),
   install_runner_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :runner),
+  install_jupyter_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :jupyter),
   toggle_status: @cluster.enabled? ? 'true': 'false',
   cluster_status: @cluster.status_name,
   cluster_status_reason: @cluster.status_reason,

db/migrate/20180511131058_create_clusters_applications_jupyter.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateClustersApplicationsJupyter < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :clusters_applications_jupyters do |t|
+      t.references :cluster, null: false, unique: true, foreign_key: { on_delete: :cascade }
+
+      t.integer :status, null: false
+      t.string :version, null: false
+      t.string :hostname
+
+      t.text :status_reason
+
+      t.timestamps_with_timezone null: false
+    end
+  end
+end

db/schema.rb

@@ -135,16 +135,13 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.boolean "clientside_sentry_enabled", default: false, null: false
     t.string "clientside_sentry_dsn"
     t.boolean "prometheus_metrics_enabled", default: true, null: false
+    t.boolean "authorized_keys_enabled", default: true, null: false
     t.boolean "help_page_hide_commercial_content", default: false
     t.string "help_page_support_url"
     t.integer "performance_bar_allowed_group_id"
     t.boolean "hashed_storage_enabled", default: false, null: false
     t.boolean "project_export_enabled", default: true, null: false
     t.boolean "auto_devops_enabled", default: false, null: false
-    t.integer "circuitbreaker_failure_count_threshold", default: 3
-    t.integer "circuitbreaker_failure_reset_time", default: 1800
-    t.integer "circuitbreaker_storage_timeout", default: 15
-    t.integer "circuitbreaker_access_retries", default: 3
     t.boolean "throttle_unauthenticated_enabled", default: false, null: false
     t.integer "throttle_unauthenticated_requests_per_period", default: 3600, null: false
     t.integer "throttle_unauthenticated_period_in_seconds", default: 3600, null: false
@@ -154,13 +151,16 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.boolean "throttle_authenticated_web_enabled", default: false, null: false
     t.integer "throttle_authenticated_web_requests_per_period", default: 7200, null: false
     t.integer "throttle_authenticated_web_period_in_seconds", default: 3600, null: false
-    t.integer "circuitbreaker_check_interval", default: 1, null: false
-    t.boolean "password_authentication_enabled_for_web"
-    t.boolean "password_authentication_enabled_for_git", default: true
+    t.integer "circuitbreaker_failure_count_threshold", default: 3
+    t.integer "circuitbreaker_failure_reset_time", default: 1800
+    t.integer "circuitbreaker_storage_timeout", default: 15
+    t.integer "circuitbreaker_access_retries", default: 3
     t.integer "gitaly_timeout_default", default: 55, null: false
     t.integer "gitaly_timeout_medium", default: 30, null: false
     t.integer "gitaly_timeout_fast", default: 10, null: false
-    t.boolean "authorized_keys_enabled", default: true, null: false
+    t.boolean "password_authentication_enabled_for_web"
+    t.boolean "password_authentication_enabled_for_git", default: true, null: false
+    t.integer "circuitbreaker_check_interval", default: 1, null: false
     t.string "auto_devops_domain"
     t.boolean "pages_domain_verification_enabled", default: true, null: false
     t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
@@ -375,12 +375,12 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.integer "project_id", null: false
     t.integer "job_id", null: false
     t.integer "file_type", null: false
-    t.integer "file_store"
     t.integer "size", limit: 8
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
     t.datetime_with_timezone "expire_at"
     t.string "file"
+    t.integer "file_store"
     t.binary "file_sha256"
   end
@@ -448,8 +448,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.integer "auto_canceled_by_id"
     t.integer "pipeline_schedule_id"
     t.integer "source"
-    t.integer "config_source"
     t.boolean "protected"
+    t.integer "config_source"
     t.integer "failure_reason"
   end
@@ -495,8 +495,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.boolean "run_untagged", default: true, null: false
     t.boolean "locked", default: false, null: false
     t.integer "access_level", default: 0, null: false
-    t.string "ip_address"
     t.integer "maximum_timeout"
+    t.string "ip_address"
     t.integer "runner_type", limit: 2, null: false
   end
@@ -635,6 +635,16 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.string "external_ip"
   end
+  create_table "clusters_applications_jupyters", force: :cascade do |t|
+    t.integer "cluster_id", null: false
+    t.integer "status", null: false
+    t.string "version", null: false
+    t.string "hostname"
+    t.text "status_reason"
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+  end
+
   create_table "clusters_applications_prometheus", force: :cascade do |t|
     t.integer "cluster_id", null: false
     t.integer "status", null: false
@@ -904,8 +914,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
   add_index "gpg_signatures", ["project_id"], name: "index_gpg_signatures_on_project_id", using: :btree
   create_table "group_custom_attributes", force: :cascade do |t|
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
     t.integer "group_id", null: false
     t.string "key", null: false
     t.string "value", null: false
@@ -987,6 +997,7 @@ ActiveRecord::Schema.define(version: 20180521171529) do
   add_index "issues", ["moved_to_id"], name: "index_issues_on_moved_to_id", where: "(moved_to_id IS NOT NULL)", using: :btree
   add_index "issues", ["project_id", "created_at", "id", "state"], name: "index_issues_on_project_id_and_created_at_and_id_and_state", using: :btree
   add_index "issues", ["project_id", "due_date", "id", "state"], name: "idx_issues_on_project_id_and_due_date_and_id_and_state_partial", where: "(due_date IS NOT NULL)", using: :btree
+  add_index "issues", ["project_id", "due_date", "id", "state"], name: "index_issues_on_project_id_and_due_date_and_id_and_state", using: :btree
   add_index "issues", ["project_id", "iid"], name: "index_issues_on_project_id_and_iid", unique: true, using: :btree
   add_index "issues", ["project_id", "updated_at", "id", "state"], name: "index_issues_on_project_id_and_updated_at_and_id_and_state", using: :btree
   add_index "issues", ["relative_position"], name: "index_issues_on_relative_position", using: :btree
@@ -1203,6 +1214,7 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.boolean "merge_when_pipeline_succeeds", default: false, null: false
     t.integer "merge_user_id"
     t.string "merge_commit_sha"
+    t.string "rebase_commit_sha"
     t.string "in_progress_merge_commit_sha"
     t.integer "lock_version"
     t.text "title_html"
@@ -1215,7 +1227,6 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.string "merge_jid"
     t.boolean "discussion_locked"
     t.integer "latest_merge_request_diff_id"
-    t.string "rebase_commit_sha"
     t.boolean "allow_maintainer_to_push"
   end
@@ -1475,8 +1486,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
   add_index "project_ci_cd_settings", ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
   create_table "project_custom_attributes", force: :cascade do |t|
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
     t.integer "project_id", null: false
     t.string "key", null: false
     t.string "value", null: false
@@ -1568,8 +1579,10 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.string "avatar"
     t.string "import_status"
     t.integer "star_count", default: 0, null: false
+    t.boolean "merge_requests_rebase_enabled", default: false, null: false
     t.string "import_type"
     t.string "import_source"
+    t.boolean "merge_requests_ff_only_enabled", default: false, null: false
     t.text "import_error"
     t.integer "ci_id"
     t.boolean "shared_runners_enabled", default: true, null: false
@@ -1585,6 +1598,7 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.boolean "only_allow_merge_if_pipeline_succeeds", default: false, null: false
     t.boolean "has_external_issue_tracker"
     t.string "repository_storage", default: "default", null: false
+    t.boolean "repository_read_only"
     t.boolean "request_access_enabled", default: false, null: false
     t.boolean "has_external_wiki"
     t.string "ci_config_path"
@@ -1599,9 +1613,6 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.datetime "last_repository_updated_at"
     t.integer "storage_version", limit: 2
     t.boolean "resolve_outdated_diff_discussions"
-    t.boolean "repository_read_only"
-    t.boolean "merge_requests_ff_only_enabled", default: false
-    t.boolean "merge_requests_rebase_enabled", default: false, null: false
     t.integer "jobs_cache_index"
     t.boolean "pages_https_only", default: true
     t.boolean "remote_mirror_available_overridden"
@@ -1945,9 +1956,9 @@ ActiveRecord::Schema.define(version: 20180521171529) do
     t.string "model_type"
     t.string "uploader", null: false
     t.datetime "created_at", null: false
+    t.integer "store"
     t.string "mount_point"
     t.string "secret"
-    t.integer "store"
   end
   add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree
@@ -2179,8 +2190,9 @@ ActiveRecord::Schema.define(version: 20180521171529) do
   add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade
   add_foreign_key "clusters", "users", on_delete: :nullify
   add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade
-  add_foreign_key "clusters_applications_ingress", "clusters", name: "fk_753a7b41c1", on_delete: :cascade
-  add_foreign_key "clusters_applications_prometheus", "clusters", name: "fk_557e773639", on_delete: :cascade
+  add_foreign_key "clusters_applications_ingress", "clusters", on_delete: :cascade
+  add_foreign_key "clusters_applications_jupyters", "clusters", on_delete: :cascade
+  add_foreign_key "clusters_applications_prometheus", "clusters", on_delete: :cascade
   add_foreign_key "clusters_applications_runners", "ci_runners", column: "runner_id", name: "fk_02de2ded36", on_delete: :nullify
   add_foreign_key "clusters_applications_runners", "clusters", on_delete: :cascade
   add_foreign_key "container_repositories", "projects"
@@ -2285,8 +2297,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
   add_foreign_key "u2f_registrations", "users"
   add_foreign_key "user_callouts", "users", on_delete: :cascade
   add_foreign_key "user_custom_attributes", "users", on_delete: :cascade
-  add_foreign_key "user_interacted_projects", "projects", name: "fk_722ceba4f7", on_delete: :cascade
-  add_foreign_key "user_interacted_projects", "users", name: "fk_0894651f08", on_delete: :cascade
+  add_foreign_key "user_interacted_projects", "projects", on_delete: :cascade
+  add_foreign_key "user_interacted_projects", "users", on_delete: :cascade
   add_foreign_key "user_synced_attributes_metadata", "users", on_delete: :cascade
   add_foreign_key "users", "application_setting_terms", column: "accepted_term_id", name: "fk_789cd90b35", on_delete: :cascade
   add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade

spec/factories/clusters/applications/helm.rb

@@ -35,5 +35,6 @@ FactoryBot.define do
     factory :clusters_applications_ingress, class: Clusters::Applications::Ingress
     factory :clusters_applications_prometheus, class: Clusters::Applications::Prometheus
     factory :clusters_applications_runner, class: Clusters::Applications::Runner
+    factory :clusters_applications_jupyter, class: Clusters::Applications::Jupyter
   end
 end

spec/fixtures/api/schemas/cluster_status.json

@@ -31,7 +31,8 @@
           }
         },
         "status_reason": { "type": ["string", "null"] },
-        "external_ip": { "type": ["string", "null"] }
+        "external_ip": { "type": ["string", "null"] },
+        "hostname": { "type": ["string", "null"] }
       },
       "required" : [ "name", "status" ]
     }

spec/models/clusters/cluster_spec.rb

@@ -234,9 +234,10 @@ describe Clusters::Cluster do
       let!(:ingress) { create(:clusters_applications_ingress, cluster: cluster) }
       let!(:prometheus) { create(:clusters_applications_prometheus, cluster: cluster) }
       let!(:runner) { create(:clusters_applications_runner, cluster: cluster) }
+      let!(:jupyter) { create(:clusters_applications_jupyter, cluster: cluster) }
       it 'returns a list of created applications' do
-        is_expected.to contain_exactly(helm, ingress, prometheus, runner)
+        is_expected.to contain_exactly(helm, ingress, prometheus, runner, jupyter)
       end
     end
   end

vendor/jupyter/values.yaml

+rbac:
+  enabled: false
+
+hub:
+  extraEnv:
+    JUPYTER_ENABLE_LAB: 1
+  extraConfig: |
+    c.KubeSpawner.cmd = ['jupyter-labhub']
+
+singleuser:
+  defaultUrl: "/lab"
+
+ingress:
+ enabled: true
+ annotations:
+   kubernetes.io/ingress.class: "nginx"