Add latest changes from gitlab-org/gitlab@master

.gitlab-ci.yml

@@ -10,7 +10,6 @@ stages:
   - review
   - qa
   - post-qa
-  - notification
   - pages
 variables:
@@ -36,7 +35,6 @@ include:
   - local: .gitlab/ci/frontend.gitlab-ci.yml
   - local: .gitlab/ci/global.gitlab-ci.yml
   - local: .gitlab/ci/memory.gitlab-ci.yml
-  - local: .gitlab/ci/notifications.gitlab-ci.yml
   - local: .gitlab/ci/pages.gitlab-ci.yml
   - local: .gitlab/ci/qa.gitlab-ci.yml
   - local: .gitlab/ci/reports.gitlab-ci.yml

.gitlab/ci/notifications.gitlab-ci.yml

-# Make sure to update all the similar conditions in other CI config files if you modify these conditions
-.if-canonical-gitlab-schedule: &if-canonical-gitlab-schedule
-  if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/)/ && $CI_PIPELINE_SOURCE == "schedule"'
-
-.notify:
-  image: ruby:2.6-alpine
-  stage: notification
-  dependencies: []
-  cache: {}
-  before_script:
-    - apk update && apk add git curl bash
-    - source scripts/utils.sh
-    - source scripts/notifications.sh
-    - install_gitlab_gem
-  variables:
-    COMMIT_NOTES_URL: "https://${CI_SERVER_HOST}/${CI_PROJECT_PATH}/commit/${CI_COMMIT_SHA}#notes-list"
-
-package-and-qa:notify-failure:
-  extends: .notify
-  rules:
-    - <<: *if-canonical-gitlab-schedule
-      when: manual # TODO: remove notify job if not necessary
-  script:
-    - 'export NOTIFICATION_MESSAGE=":skull_and_crossbones: Scheduled QA against master failed! :skull_and_crossbones: See ${CI_PIPELINE_URL}. For downstream pipelines, see ${COMMIT_NOTES_URL}"'
-    - 'notify_on_job_failure package-and-qa qa-master "${NOTIFICATION_MESSAGE}" ci_failing'
-  needs: ["package-and-qa"]
-  allow_failure: true

GITALY_SERVER_VERSION

-1.78.0
+1.79.0

app/assets/javascripts/clusters/clusters_bundle.js

@@ -53,6 +53,7 @@ export default class Clusters {
       helpPath,
       ingressHelpPath,
       ingressDnsHelpPath,
+      ingressModSecurityHelpPath,
       environmentsHelpPath,
       clustersHelpPath,
       deployBoardsHelpPath,
@@ -69,6 +70,7 @@ export default class Clusters {
       helpPath,
       ingressHelpPath,
       ingressDnsHelpPath,
+      ingressModSecurityHelpPath,
       environmentsHelpPath,
       clustersHelpPath,
       deployBoardsHelpPath,
@@ -169,6 +171,7 @@ export default class Clusters {
             ingressHelpPath: this.state.ingressHelpPath,
             managePrometheusPath: this.state.managePrometheusPath,
             ingressDnsHelpPath: this.state.ingressDnsHelpPath,
+            ingressModSecurityHelpPath: this.state.ingressModSecurityHelpPath,
             cloudRunHelpPath: this.state.cloudRunHelpPath,
             providerType: this.state.providerType,
             preInstalledKnative: this.state.preInstalledKnative,

app/assets/javascripts/clusters/components/applications.vue

@@ -56,6 +56,11 @@ export default {
       required: false,
       default: '',
     },
+    ingressModSecurityHelpPath: {
+      type: String,
+      required: false,
+      default: '',
+    },
     cloudRunHelpPath: {
       type: String,
       required: false,
@@ -112,6 +117,9 @@ export default {
     ingressInstalled() {
       return this.applications.ingress.status === APPLICATION_STATUS.INSTALLED;
     },
+    ingressEnableModsecurity() {
+      return this.applications.ingress.modsecurity_enabled;
+    },
     ingressExternalEndpoint() {
       return this.applications.ingress.externalIp || this.applications.ingress.externalHostname;
     },
@@ -127,6 +135,18 @@ export default {
     enableClusterApplicationElasticStack() {
       return gon.features && gon.features.enableClusterApplicationElasticStack;
     },
+    ingressModSecurityDescription() {
+      const escapedUrl = _.escape(this.ingressModSecurityHelpPath);
+
+      return sprintf(
+        s__('ClusterIntegration|Learn more about %{startLink}ModSecurity%{endLink}'),
+        {
+          startLink: `<a href="${escapedUrl}" target="_blank" rel="noopener noreferrer">`,
+          endLink: '</a>',
+        },
+        false,
+      );
+    },
     ingressDescription() {
       return sprintf(
         _.escape(
@@ -135,9 +155,9 @@ export default {
           ),
         ),
         {
-          pricingLink: `<strong><a href="https://cloud.google.com/compute/pricing#lb"
+          pricingLink: `<a href="https://cloud.google.com/compute/pricing#lb"
               target="_blank" rel="noopener noreferrer">
-              ${_.escape(s__('ClusterIntegration|pricing'))}</a></strong>`,
+              ${_.escape(s__('ClusterIntegration|pricing'))}</a>`,
         },
         false,
       );
@@ -311,6 +331,9 @@ Crossplane runs inside your Kubernetes cluster and supports secure connectivity
         :request-reason="applications.ingress.requestReason"
         :installed="applications.ingress.installed"
         :install-failed="applications.ingress.installFailed"
+        :install-application-request-params="{
+          modsecurity_enabled: applications.ingress.modsecurity_enabled,
+        }"
         :uninstallable="applications.ingress.uninstallable"
         :uninstall-successful="applications.ingress.uninstallSuccessful"
         :uninstall-failed="applications.ingress.uninstallFailed"
@@ -326,6 +349,26 @@ Crossplane runs inside your Kubernetes cluster and supports secure connectivity
             }}
           </p>
+          <template>
+            <div class="form-group">
+              <div class="form-check form-check-inline">
+                <input
+                  v-model="applications.ingress.modsecurity_enabled"
+                  :disabled="ingressInstalled"
+                  type="checkbox"
+                  autocomplete="off"
+                  class="form-check-input"
+                />
+                <label class="form-check-label label-bold" for="ingress-enable-modsecurity">
+                  {{ s__('ClusterIntegration|Enable Web Application Firewall') }}
+                </label>
+              </div>
+              <p class="form-text text-muted">
+                <strong v-html="ingressModSecurityDescription"></strong>
+              </p>
+            </div>
+          </template>
+
           <template v-if="ingressInstalled">
             <div class="form-group">
               <label for="ingress-endpoint">{{ s__('ClusterIntegration|Ingress Endpoint') }}</label>
@@ -375,7 +418,9 @@ Crossplane runs inside your Kubernetes cluster and supports secure connectivity
             </p>
           </template>
           <template v-if="!ingressInstalled">
-            <div class="bs-callout bs-callout-info" v-html="ingressDescription"></div>
+            <div class="bs-callout bs-callout-info">
+              <strong v-html="ingressDescription"></strong>
+            </div>
           </template>
         </div>
       </application-row>

app/assets/javascripts/clusters/stores/clusters_store.js

@@ -52,6 +52,7 @@ export default class ClusterStore {
         ingress: {
           ...applicationInitialState,
           title: s__('ClusterIntegration|Ingress'),
+          modsecurity_enabled: false,
           externalIp: null,
           externalHostname: null,
         },
@@ -108,6 +109,7 @@ export default class ClusterStore {
     helpPath,
     ingressHelpPath,
     ingressDnsHelpPath,
+    ingressModSecurityHelpPath,
     environmentsHelpPath,
     clustersHelpPath,
     deployBoardsHelpPath,
@@ -116,6 +118,7 @@ export default class ClusterStore {
     this.state.helpPath = helpPath;
     this.state.ingressHelpPath = ingressHelpPath;
     this.state.ingressDnsHelpPath = ingressDnsHelpPath;
+    this.state.ingressModSecurityHelpPath = ingressModSecurityHelpPath;
     this.state.environmentsHelpPath = environmentsHelpPath;
     this.state.clustersHelpPath = clustersHelpPath;
     this.state.deployBoardsHelpPath = deployBoardsHelpPath;
@@ -207,6 +210,8 @@ export default class ClusterStore {
       if (appId === INGRESS) {
         this.state.applications.ingress.externalIp = serverAppEntry.external_ip;
         this.state.applications.ingress.externalHostname = serverAppEntry.external_hostname;
+        this.state.applications.ingress.modsecurity_enabled =
+          serverAppEntry.modsecurity_enabled || this.state.applications.ingress.modsecurity_enabled;
       } else if (appId === CERT_MANAGER) {
         this.state.applications.cert_manager.email =
           this.state.applications.cert_manager.email || serverAppEntry.email;

app/controllers/clusters/applications_controller.rb

@@ -47,7 +47,7 @@ class Clusters::ApplicationsController < Clusters::BaseController
   end
   def cluster_application_params
-    params.permit(:application, :hostname, :kibana_hostname, :email, :stack)
+    params.permit(:application, :hostname, :kibana_hostname, :email, :stack, :modsecurity_enabled)
   end
   def cluster_application_destroy_params

app/models/clusters/applications/ingress.rb

@@ -14,6 +14,7 @@ module Clusters
       include AfterCommitQueue
       default_value_for :ingress_type, :nginx
+      default_value_for :modsecurity_enabled, false
       default_value_for :version, VERSION
       enum ingress_type: {
@@ -73,7 +74,7 @@ module Clusters
       private
       def specification
-        return {} unless Feature.enabled?(:ingress_modsecurity)
+        return {} unless modsecurity_enabled
         {
           "controller" => {

app/serializers/cluster_application_entity.rb

@@ -11,6 +11,7 @@ class ClusterApplicationEntity < Grape::Entity
   expose :kibana_hostname, if: -> (e, _) { e.respond_to?(:kibana_hostname) }
   expose :email, if: -> (e, _) { e.respond_to?(:email) }
   expose :stack, if: -> (e, _) { e.respond_to?(:stack) }
+  expose :modsecurity_enabled, if: -> (e, _) { e.respond_to?(:modsecurity_enabled) }
   expose :update_available?, as: :update_available, if: -> (e, _) { e.respond_to?(:update_available?) }
   expose :can_uninstall?, as: :can_uninstall
 end

app/services/clusters/applications/base_service.rb

@@ -31,6 +31,10 @@ module Clusters
             application.stack = params[:stack]
           end
+          if application.has_attribute?(:modsecurity_enabled)
+            application.modsecurity_enabled = params[:modsecurity_enabled] || false
+          end
+
           if application.respond_to?(:oauth_application)
             application.oauth_application = create_oauth_application(application, request)
           end

app/views/clusters/clusters/show.html.haml

@@ -30,6 +30,7 @@
   help_path: help_page_path('user/project/clusters/index.md', anchor: 'installing-applications'),
   ingress_help_path: help_page_path('user/project/clusters/index.md', anchor: 'getting-the-external-endpoint'),
   ingress_dns_help_path: help_page_path('user/project/clusters/index.md', anchor: 'manually-determining-the-external-endpoint'),
+  ingress_mod_security_help_path: help_page_path('user/clusters/applications.md', anchor: 'web-application-firewall-modsecurity'),
   environments_help_path: help_page_path('ci/environments', anchor: 'defining-environments'),
   clusters_help_path: help_page_path('user/project/clusters/index.md', anchor: 'deploying-to-a-kubernetes-cluster'),
   deploy_boards_help_path: help_page_path('user/project/deploy_boards.html', anchor: 'enabling-deploy-boards'),

app/workers/all_queues.yml

@@ -187,3 +187,4 @@
 - project_daily_statistics
 - create_evidence
 - group_export
+- self_monitoring_project_create

app/workers/self_monitoring_project_create_worker.rb

+# frozen_string_literal: true
+
+class SelfMonitoringProjectCreateWorker
+  include ApplicationWorker
+  include ExclusiveLeaseGuard
+
+  # This worker falls under Self-monitoring with Monitor::APM group. However,
+  # self-monitoring is not classified as a feature category but rather as
+  # Other Functionality. Metrics seems to be the closest feature_category for
+  # this worker.
+  feature_category :metrics
+
+  LEASE_TIMEOUT = 15.minutes.to_i
+
+  EXCLUSIVE_LEASE_KEY = 'self_monitoring_service_creation_deletion'
+
+  def perform
+    try_obtain_lease do
+      Gitlab::DatabaseImporters::SelfMonitoring::Project::CreateService.new.execute
+    end
+  end
+
+  # @param job_id [String]
+  #   Job ID that is used to construct the cache keys.
+  # @return [Hash]
+  #   Returns true if the job is enqueued or in progress and false otherwise.
+  def self.in_progress?(job_id)
+    Gitlab::SidekiqStatus.job_status(Array.wrap(job_id)).first
+  end
+
+  private
+
+  def lease_key
+    EXCLUSIVE_LEASE_KEY
+  end
+
+  def lease_timeout
+    LEASE_TIMEOUT
+  end
+end

changelogs/unreleased/39140-toggle-modsecurity-enabled-for-managed-ingress.yml

+---
+title: Add enable_modsecurity setting to managed ingress
+merge_request: 21966
+author:
+type: added

changelogs/unreleased/gitaly-version-v1.79.0.yml

+---
+title: Upgrade to Gitaly v1.79.0
+merge_request: 22515
+author:
+type: changed

config/sidekiq_queues.yml

@@ -99,6 +99,7 @@
   - [chaos, 2]
   - [create_evidence, 2]
   - [group_export, 1]
+  - [self_monitoring_project_create, 2]
   # EE-specific queues
   - [analytics, 1]

doc/development/pipelines.md

@@ -264,10 +264,6 @@ subgraph "`qa` stage"
     dast -.-> |needs and depends on| G;
     end
-subgraph "`notification` stage"
-    NOTIFICATION2["package-and-qa:notify-failure<br>(manual)"] -.-> |needs| Q;
-    end
-
 subgraph "`post-test` stage"
     M
     end

doc/user/clusters/applications.md

@@ -248,10 +248,10 @@ use an A record. If your external endpoint is a hostname, use a CNAME record.
 #### Web Application Firewall (ModSecurity)
-> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/65192) in GitLab 12.3 (enabled using `ingress_modsecurity` [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development)).
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/21966) in GitLab 12.7.
 Out of the box, GitLab provides you real-time security monitoring with
-[`modsecurity`](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#modsecurity)
+[ModSecurity](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#modsecurity).
 Modsecurity is a toolkit for real-time web application monitoring, logging,
 and access control. With GitLab's offering, the [OWASP's Core Rule Set](https://www.modsecurity.org/CRS/Documentation/), which provides generic attack detection capabilities,
@@ -267,22 +267,18 @@ This feature:
   kubectl -n gitlab-managed-apps exec -it $(kubectl get pods -n gitlab-managed-apps | grep 'ingress-controller' | awk '{print $1}') -- tail -f /var/log/modsec/audit.log
   ```
-There is a small performance overhead by enabling `modsecurity`. If this is
-considered significant for your application, you can either:
-- Disable ModSecurity's rule engine for your deployed application by setting
-  [the deployment variable](../../topics/autodevops/index.md)
-  `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` to `Off`. This will prevent ModSecurity from
-  processing any requests for the given application or environment.
-- Toggle the feature flag to false by running the following command within your
-  instance's Rails console:
-  ```ruby
-  Feature.disable(:ingress_modsecurity)
-  ```
-
-Once disabled, you must [uninstall](#uninstalling-applications) and reinstall your Ingress
-application for the changes to take effect.
+To enable ModSecurity, check the **Enable Web Application Firewall** checkbox
+when installing your [Ingress application](#ingress).
+There is a small performance overhead by enabling ModSecurity. If this is
+considered significant for your application, you can disable ModSecurity's
+rule engine for your deployed application by setting
+[the deployment variable](../../topics/autodevops/index.md)
+`AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` to `Off`. This will prevent ModSecurity
+from processing any requests for the given application or environment.
+To permanently disable it, you must [uninstall](#uninstalling-applications) and
+reinstall your Ingress application for the changes to take effect.
 ### JupyterHub

lib/gitlab/database_importers/self_monitoring/helpers.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module DatabaseImporters
+    module SelfMonitoring
+      module Helpers
+        def application_settings
+          @application_settings ||= ApplicationSetting.current_without_cache
+        end
+
+        def project_created?
+          self_monitoring_project.present?
+        end
+
+        def self_monitoring_project
+          application_settings.instance_administration_project
+        end
+
+        def self_monitoring_project_id
+          application_settings.instance_administration_project_id
+        end
+      end
+    end
+  end
+end

lib/gitlab/database_importers/self_monitoring/project/create_service.rb

@@ -6,38 +6,26 @@ module Gitlab
       module Project
         class CreateService < ::BaseService
           include Stepable
-
-          STEPS_ALLOWED_TO_FAIL = [
-            :validate_application_settings, :validate_project_created, :validate_admins
-          ].freeze
+          include SelfMonitoring::Helpers
           VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL
           PROJECT_NAME = 'GitLab Instance Administration'
           steps :validate_application_settings,
-            :validate_project_created,
             :validate_admins,
             :create_group,
             :create_project,
             :save_project_id,
             :add_group_members,
-            :add_prometheus_manual_configuration
+            :add_prometheus_manual_configuration,
+            :track_event
           def initialize
             super(nil)
           end
-          def execute!
-            result = execute_steps
-            if result[:status] == :success
-              ::Gitlab::Tracking.event("self_monitoring", "project_created")
-              result
-            elsif STEPS_ALLOWED_TO_FAIL.include?(result[:last_step])
-              ::Gitlab::Tracking.event("self_monitoring", "project_created")
-              success
-            else
-              raise StandardError, result[:message]
-            end
+          def execute
+            execute_steps
           end
           private
@@ -49,13 +37,6 @@ module Gitlab
             error(_('No application_settings found'))
           end
-          def validate_project_created(result)
-            return success(result) unless project_created?
-
-            log_error('Project already created')
-            error(_('Project already created'))
-          end
-
           def validate_admins(result)
             unless instance_admins.any?
               log_error('No active admin user found')
@@ -68,7 +49,7 @@ module Gitlab
           def create_group(result)
             if project_created?
               log_info(_('Instance administrators group already exists'))
-              result[:group] = application_settings.instance_administration_project.owner
+              result[:group] = self_monitoring_project.owner
               return success(result)
             end
@@ -84,7 +65,7 @@ module Gitlab
           def create_project(result)
             if project_created?
               log_info('Instance administration project already exists')
-              result[:project] = application_settings.instance_administration_project
+              result[:project] = self_monitoring_project
               return success(result)
             end
@@ -99,7 +80,7 @@ module Gitlab
           end
           def save_project_id(result)
-            return success if project_created?
+            return success(result) if project_created?
             response = application_settings.update(
               instance_administration_project_id: result[:project].id
@@ -140,12 +121,10 @@ module Gitlab
             success(result)
           end
-          def application_settings
-            @application_settings ||= ApplicationSetting.current_without_cache
-          end
-          def project_created?
-            application_settings.instance_administration_project.present?
+          def track_event(result)
+            ::Gitlab::Tracking.event("self_monitoring", "project_created")
+            success(result)
           end
           def parse_url(uri_string)