Add latest changes from gitlab-org/gitlab@master

app/models/clusters/applications/ingress.rb

@@ -78,12 +78,42 @@ module Clusters
           "controller" => {
             "config" => {
               "enable-modsecurity" => "true",
-              "enable-owasp-modsecurity-crs" => "true"
-            }
+              "enable-owasp-modsecurity-crs" => "true",
+              "modsecurity.conf" => modsecurity_config_content
+            },
+            "extraVolumeMounts" => [
+              {
+                "name" => "modsecurity-template-volume",
+                "mountPath" => "/etc/nginx/modsecurity/modsecurity.conf",
+                "subPath" => "modsecurity.conf"
+              }
+            ],
+            "extraVolumes" => [
+              {
+                "name" => "modsecurity-template-volume",
+                "configMap" => {
+                  "name" => "ingress-nginx-ingress-controller",
+                  "items" => [
+                    {
+                      "key" => "modsecurity.conf",
+                      "path" => "modsecurity.conf"
+                    }
+                  ]
+                }
+              }
+            ]
           }
         }
       end
+      def modsecurity_config_content
+        File.read(modsecurity_config_file_path)
+      end
+
+      def modsecurity_config_file_path
+        Rails.root.join('vendor', 'ingress', 'modsecurity.conf')
+      end
+
       def content_values
         YAML.load_file(chart_values_file).deep_merge!(specification)
       end

changelogs/unreleased/13055-show-the-sync-information-for-design-repositories.yml

+---
+title: 'Geo: Add resigns-related fields to Geo Node Status table'
+merge_request: 18379
+author:
+type: other

changelogs/unreleased/8558-use-custom-modsecurity-ingress-config.yml

+---
+title: Add modsecurity template for ingress-controller
+merge_request: 18485
+author:
+type: changed

db/migrate/20191009100244_add_geo_design_repository_counters.rb

+# frozen_string_literal: true
+
+class AddGeoDesignRepositoryCounters < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    change_table :geo_node_statuses do |t|
+      t.column :design_repositories_count, :integer
+      t.column :design_repositories_synced_count, :integer
+      t.column :design_repositories_failed_count, :integer
+      t.column :design_repositories_registry_count, :integer
+    end
+  end
+end

db/schema.rb

@@ -1650,6 +1650,10 @@ ActiveRecord::Schema.define(version: 2019_10_17_045817) do
     t.integer "container_repositories_synced_count"
     t.integer "container_repositories_failed_count"
     t.integer "container_repositories_registry_count"
+    t.integer "design_repositories_count"
+    t.integer "design_repositories_synced_count"
+    t.integer "design_repositories_failed_count"
+    t.integer "design_repositories_registry_count"
     t.index ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true
   end

doc/administration/geo/replication/using_a_geo_server.md

@@ -10,8 +10,12 @@ Example of the output you will see when pushing to a **secondary** node:
 ```bash
 $ git push
-> GitLab: You're pushing to a Geo secondary.
-> GitLab: We'll help you by proxying this request to the primary: ssh://git@primary.geo/user/repo.git
+remote:
+remote: You're pushing to a Geo secondary. We'll help you by proxying this
+remote: request to the primary:
+remote:
+remote:   ssh://git@primary.geo/user/repo.git
+remote:
 Everything up-to-date
 ```

doc/api/geo_nodes.md

@@ -237,6 +237,10 @@ Example response:
     "container_repositories_synced_count": nil,
     "container_repositories_failed_count": nil,
     "container_repositories_synced_in_percentage": "0.00%",
+    "design_repositories_count": 3,
+    "design_repositories_synced_count": nil,
+    "design_repositories_failed_count": nil,
+    "design_repositories_synced_in_percentage": "0.00%",
     "projects_count": 41,
     "repositories_failed_count": nil,
     "repositories_synced_count": nil,
@@ -304,6 +308,10 @@ Example response:
     "container_repositories_synced_count": nil,
     "container_repositories_failed_count": nil,
     "container_repositories_synced_in_percentage": "0.00%",
+    "design_repositories_count": 3,
+    "design_repositories_synced_count": nil,
+    "design_repositories_failed_count": nil,
+    "design_repositories_synced_in_percentage": "0.00%",
     "projects_count": 41,
     "repositories_failed_count": 1,
     "repositories_synced_count": 40,
@@ -387,6 +395,10 @@ Example response:
   "container_repositories_synced_count": nil,
   "container_repositories_failed_count": nil,
   "container_repositories_synced_in_percentage": "0.00%",
+  "design_repositories_count": 3,
+  "design_repositories_synced_count": nil,
+  "design_repositories_failed_count": nil,
+  "design_repositories_synced_in_percentage": "0.00%",
   "projects_count": 41,
   "repositories_failed_count": 1,
   "repositories_synced_count": 40,

lib/api/internal/base.rb

@@ -77,7 +77,7 @@ module API
             response_with_status(**payload)
           when ::Gitlab::GitAccessResult::CustomAction
-            response_with_status(code: 300, message: check_result.message, payload: check_result.payload)
+            response_with_status(code: 300, payload: check_result.payload, gl_console_messages: check_result.console_messages)
           else
             response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR)
           end

lib/gitlab/analytics/cycle_analytics/base_query_builder.rb

@@ -68,3 +68,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder')

lib/gitlab/analytics/cycle_analytics/records_fetcher.rb

@@ -130,3 +130,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::RecordsFetcher')

lib/gitlab/git_access_result/custom_action.rb

@@ -3,7 +3,7 @@
 module Gitlab
   module GitAccessResult
     class CustomAction
-      attr_reader :payload, :message
+      attr_reader :payload, :console_messages
       # Example of payload:
       #
@@ -16,9 +16,9 @@ module Gitlab
       #   }
       # }
       #
-      def initialize(payload, message)
+      def initialize(payload, console_messages)
         @payload = payload
-        @message = message
+        @console_messages = console_messages
       end
     end
   end

locale/gitlab.pot

@@ -7493,6 +7493,9 @@ msgstr ""
 msgid "GeoNodes|Data replication lag"
 msgstr ""
+msgid "GeoNodes|Design repositories"
+msgstr ""
+
 msgid "GeoNodes|Does not match the primary storage configuration"
 msgstr ""

spec/models/clusters/applications/ingress_spec.rb

@@ -156,6 +156,15 @@ describe Clusters::Applications::Ingress do
       it 'includes modsecurity core ruleset enablement' do
         expect(subject.values).to include("enable-owasp-modsecurity-crs: 'true'")
       end
+
+      it 'includes modsecurity.conf content' do
+        expect(subject.values).to include('modsecurity.conf')
+        # Includes file content from Ingress#modsecurity_config_content
+        expect(subject.values).to include('SecAuditLog')
+
+        expect(subject.values).to include('extraVolumes')
+        expect(subject.values).to include('extraVolumeMounts')
+      end
     end
     context 'when ingress_modsecurity is disabled' do
@@ -172,6 +181,15 @@ describe Clusters::Applications::Ingress do
       it 'excludes modsecurity core ruleset enablement' do
         expect(subject.values).not_to include('enable-owasp-modsecurity-crs')
       end
+
+      it 'excludes modsecurity.conf content' do
+        expect(subject.values).not_to include('modsecurity.conf')
+        # Excludes file content from Ingress#modsecurity_config_content
+        expect(subject.values).not_to include('SecAuditLog')
+
+        expect(subject.values).not_to include('extraVolumes')
+        expect(subject.values).not_to include('extraVolumeMounts')
+      end
     end
   end
 end

spec/requests/api/internal/base_spec.rb

@@ -407,7 +407,6 @@ describe API::Internal::Base do
     context "custom action" do
       let(:access_checker) { double(Gitlab::GitAccess) }
-      let(:message) { 'CustomActionError message' }
       let(:payload) do
         {
           'action' => 'geo_proxy_to_primary',
@@ -418,8 +417,8 @@ describe API::Internal::Base do
           }
         }
       end
-
-      let(:custom_action_result) { Gitlab::GitAccessResult::CustomAction.new(payload, message) }
+      let(:console_messages) { ['informational message'] }
+      let(:custom_action_result) { Gitlab::GitAccessResult::CustomAction.new(payload, console_messages) }
       before do
         project.add_guest(user)
@@ -446,8 +445,8 @@ describe API::Internal::Base do
           expect(response).to have_gitlab_http_status(300)
           expect(json_response['status']).to be_truthy
-          expect(json_response['message']).to eql(message)
           expect(json_response['payload']).to eql(payload)
+          expect(json_response['gl_console_messages']).to eql(console_messages)
           expect(user.reload.last_activity_on).to be_nil
         end
       end

vendor/ingress/modsecurity.conf

+# -- GitLab Customization ----------------------------------------------
+# Based on https://github.com/SpiderLabs/ModSecurity/blob/v3.0.3/modsecurity.conf-recommended
+# Our base modsecurity.conf includes some minor customization:
+#  - `SecRuleEngine` is disabled, defaulting to `DetectionOnly`. Overridable at project-level
+#  - `SecAuditLogType` is disabled, defaulting to `Serial`. Overridable at project-level
+#  - `SecStatusEngine` is disabled, to disallow usage reporting
+#
+# ----------------------------------------------------------------------------
+
+# -- Rule engine initialization ----------------------------------------------
+
+# Enable ModSecurity, attaching it to every transaction. Use detection
+# only to start with, because that minimises the chances of post-installation
+# disruption.
+#
+# SecRuleEngine DetectionOnly
+
+
+# -- Request body handling ---------------------------------------------------
+
+# Allow ModSecurity to access request bodies. If you don't, ModSecurity
+# won't be able to see any POST parameters, which opens a large security
+# hole for attackers to exploit.
+#
+SecRequestBodyAccess On
+
+
+# Enable XML request body parser.
+# Initiate XML Processor in case of xml content-type
+#
+SecRule REQUEST_HEADERS:Content-Type "(?:application(?:/soap\+|/)|text/)xml" \
+     "id:'200000',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=XML"
+
+# Enable JSON request body parser.
+# Initiate JSON Processor in case of JSON content-type; change accordingly
+# if your application does not use 'application/json'
+#
+SecRule REQUEST_HEADERS:Content-Type "application/json" \
+     "id:'200001',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=JSON"
+
+# Maximum request body size we will accept for buffering. If you support
+# file uploads then the value given on the first line has to be as large
+# as the largest file you are willing to accept. The second value refers
+# to the size of data, with files excluded. You want to keep that value as
+# low as practical.
+#
+SecRequestBodyLimit 13107200
+SecRequestBodyNoFilesLimit 131072
+
+# What do do if the request body size is above our configured limit.
+# Keep in mind that this setting will automatically be set to ProcessPartial
+# when SecRuleEngine is set to DetectionOnly mode in order to minimize
+# disruptions when initially deploying ModSecurity.
+#
+SecRequestBodyLimitAction Reject
+
+# Verify that we've correctly processed the request body.
+# As a rule of thumb, when failing to process a request body
+# you should reject the request (when deployed in blocking mode)
+# or log a high-severity alert (when deployed in detection-only mode).
+#
+SecRule REQBODY_ERROR "!@eq 0" \
+"id:'200002', phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2"
+
+# By default be strict with what we accept in the multipart/form-data
+# request body. If the rule below proves to be too strict for your
+# environment consider changing it to detection-only. You are encouraged
+# _not_ to remove it altogether.
+#
+SecRule MULTIPART_STRICT_ERROR "!@eq 0" \
+"id:'200003',phase:2,t:none,log,deny,status:400, \
+msg:'Multipart request body failed strict validation: \
+PE %{REQBODY_PROCESSOR_ERROR}, \
+BQ %{MULTIPART_BOUNDARY_QUOTED}, \
+BW %{MULTIPART_BOUNDARY_WHITESPACE}, \
+DB %{MULTIPART_DATA_BEFORE}, \
+DA %{MULTIPART_DATA_AFTER}, \
+HF %{MULTIPART_HEADER_FOLDING}, \
+LF %{MULTIPART_LF_LINE}, \
+SM %{MULTIPART_MISSING_SEMICOLON}, \
+IQ %{MULTIPART_INVALID_QUOTING}, \
+IP %{MULTIPART_INVALID_PART}, \
+IH %{MULTIPART_INVALID_HEADER_FOLDING}, \
+FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'"
+
+# Did we see anything that might be a boundary?
+#
+# Here is a short description about the ModSecurity Multipart parser: the
+# parser returns with value 0, if all "boundary-like" line matches with
+# the boundary string which given in MIME header. In any other cases it returns
+# with different value, eg. 1 or 2.
+#
+# The RFC 1341 descript the multipart content-type and its syntax must contains
+# only three mandatory lines (above the content):
+# * Content-Type: multipart/mixed; boundary=BOUNDARY_STRING
+# * --BOUNDARY_STRING
+# * --BOUNDARY_STRING--
+#
+# First line indicates, that this is a multipart content, second shows that
+# here starts a part of the multipart content, third shows the end of content.
+#
+# If there are any other lines, which starts with "--", then it should be
+# another boundary id - or not.
+#
+# After 3.0.3, there are two kinds of types of boundary errors: strict and permissive.
+#
+# If multipart content contains the three necessary lines with correct order, but
+# there are one or more lines with "--", then parser returns with value 2 (non-zero).
+#
+# If some of the necessary lines (usually the start or end) misses, or the order
+# is wrong, then parser returns with value 1 (also a non-zero).
+#
+# You can choose, which one is what you need. The example below contains the
+# 'strict' mode, which means if there are any lines with start of "--", then
+# ModSecurity blocked the content. But the next, commented example contains
+# the 'permissive' mode, then you check only if the necessary lines exists in
+# correct order. Whit this, you can enable to upload PEM files (eg "----BEGIN.."),
+# or other text files, which contains eg. HTTP headers.
+#
+# The difference is only the operator - in strict mode (first) the content blocked
+# in case of any non-zero value. In permissive mode (second, commented) the
+# content blocked only if the value is explicit 1. If it 0 or 2, the content will
+# allowed.
+#
+
+#
+# See #1747 and #1924 for further information on the possible values for
+# MULTIPART_UNMATCHED_BOUNDARY.
+#
+SecRule MULTIPART_UNMATCHED_BOUNDARY "@eq 1" \
+    "id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'"
+
+
+# PCRE Tuning
+# We want to avoid a potential RegEx DoS condition
+#
+SecPcreMatchLimit 1000
+SecPcreMatchLimitRecursion 1000
+
+# Some internal errors will set flags in TX and we will need to look for these.
+# All of these are prefixed with "MSC_".  The following flags currently exist:
+#
+# MSC_PCRE_LIMITS_EXCEEDED: PCRE match limits were exceeded.
+#
+SecRule TX:/^MSC_/ "!@streq 0" \
+        "id:'200005',phase:2,t:none,deny,msg:'ModSecurity internal error flagged: %{MATCHED_VAR_NAME}'"
+
+
+# -- Response body handling --------------------------------------------------
+
+# Allow ModSecurity to access response bodies. 
+# You should have this directive enabled in order to identify errors
+# and data leakage issues.
+# 
+# Do keep in mind that enabling this directive does increases both
+# memory consumption and response latency.
+#
+SecResponseBodyAccess On
+
+# Which response MIME types do you want to inspect? You should adjust the
+# configuration below to catch documents but avoid static files
+# (e.g., images and archives).
+#
+SecResponseBodyMimeType text/plain text/html text/xml
+
+# Buffer response bodies of up to 512 KB in length.
+SecResponseBodyLimit 524288
+
+# What happens when we encounter a response body larger than the configured
+# limit? By default, we process what we have and let the rest through.
+# That's somewhat less secure, but does not break any legitimate pages.
+#
+SecResponseBodyLimitAction ProcessPartial
+
+
+# -- Filesystem configuration ------------------------------------------------
+
+# The location where ModSecurity stores temporary files (for example, when
+# it needs to handle a file upload that is larger than the configured limit).
+# 
+# This default setting is chosen due to all systems have /tmp available however, 
+# this is less than ideal. It is recommended that you specify a location that's private.
+#
+SecTmpDir /tmp/
+
+# The location where ModSecurity will keep its persistent data.  This default setting 
+# is chosen due to all systems have /tmp available however, it
+# too should be updated to a place that other users can't access.
+#
+SecDataDir /tmp/
+
+
+# -- File uploads handling configuration -------------------------------------
+
+# The location where ModSecurity stores intercepted uploaded files. This
+# location must be private to ModSecurity. You don't want other users on
+# the server to access the files, do you?
+#
+#SecUploadDir /opt/modsecurity/var/upload/
+
+# By default, only keep the files that were determined to be unusual
+# in some way (by an external inspection script). For this to work you
+# will also need at least one file inspection rule.
+#
+#SecUploadKeepFiles RelevantOnly
+
+# Uploaded files are by default created with permissions that do not allow
+# any other user to access them. You may need to relax that if you want to
+# interface ModSecurity to an external program (e.g., an anti-virus).
+#
+#SecUploadFileMode 0600
+
+
+# -- Debug log configuration -------------------------------------------------
+
+# The default debug log configuration is to duplicate the error, warning
+# and notice messages from the error log.
+#
+#SecDebugLog /opt/modsecurity/var/log/debug.log
+#SecDebugLogLevel 3
+
+
+# -- Audit log configuration -------------------------------------------------
+
+# Log the transactions that are marked by a rule, as well as those that
+# trigger a server error (determined by a 5xx or 4xx, excluding 404,  
+# level response status codes).
+#
+SecAuditEngine RelevantOnly
+SecAuditLogRelevantStatus "^(?:5|4(?!04))"
+
+# Log everything we know about a transaction.
+SecAuditLogParts ABIJDEFHZ
+
+# Use a single file for logging. This is much easier to look at, but
+# assumes that you will use the audit log only ocassionally.
+#
+# SecAuditLogType Serial
+SecAuditLog /var/log/modsec_audit.log
+
+# Specify the path for concurrent audit logging.
+#SecAuditLogStorageDir /opt/modsecurity/var/audit/
+
+
+# -- Miscellaneous -----------------------------------------------------------
+
+# Use the most commonly used application/x-www-form-urlencoded parameter
+# separator. There's probably only one application somewhere that uses
+# something else so don't expect to change this value.
+#
+SecArgumentSeparator &
+
+# Settle on version 0 (zero) cookies, as that is what most applications
+# use. Using an incorrect cookie version may open your installation to
+# evasion attacks (against the rules that examine named cookies).
+#
+SecCookieFormat 0
+
+# Specify your Unicode Code Point.
+# This mapping is used by the t:urlDecodeUni transformation function
+# to properly map encoded data to your language. Properly setting
+# these directives helps to reduce false positives and negatives.
+#
+SecUnicodeMapFile unicode.mapping 20127
+
+# Improve the quality of ModSecurity by sharing information about your
+# current ModSecurity version and dependencies versions.
+# The following information will be shared: ModSecurity version,
+# Web Server version, APR version, PCRE version, Lua version, Libxml2
+# version, Anonymous unique id for host.
+# SecStatusEngine On
+
+