Add latest changes from gitlab-org/gitlab@master

app/helpers/application_settings_helper.rb

@@ -176,6 +176,7 @@ module ApplicationSettingsHelper
       :container_registry_token_expire_delay,
       :default_artifacts_expire_in,
       :default_branch_protection,
+      :default_ci_config_path,
       :default_group_visibility,
       :default_project_creation,
       :default_project_visibility,

app/models/application_setting.rb

@@ -297,6 +297,12 @@ class ApplicationSetting < ApplicationRecord
                  pass: :external_auth_client_key_pass,
                  if: -> (setting) { setting.external_auth_client_cert.present? }
+  validates :default_ci_config_path,
+    format: { without: %r{(\.{2}|\A/)},
+              message: N_('cannot include leading slash or directory traversal.') },
+    length: { maximum: 255 },
+    allow_blank: true
+
   attr_encrypted :asset_proxy_secret_key,
                  mode: :per_attribute_iv,
                  key: Settings.attr_encrypted_db_key_base_truncated,

app/models/application_setting_implementation.rb

@@ -42,6 +42,7 @@ module ApplicationSettingImplementation
         container_registry_token_expire_delay: 5,
         default_artifacts_expire_in: '30 days',
         default_branch_protection: Settings.gitlab['default_branch_protection'],
+        default_ci_config_path: nil,
         default_group_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         default_project_creation: Settings.gitlab['default_project_creation'],
         default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],

app/models/clusters/applications/ingress.rb

@@ -21,6 +21,7 @@ module Clusters
       }
       FETCH_IP_ADDRESS_DELAY = 30.seconds
+      MODSEC_SIDECAR_INITIAL_DELAY_SECONDS = 10
       state_machine :status do
         after_transition any => [:installed] do |application|
@@ -81,11 +82,39 @@ module Clusters
               "enable-owasp-modsecurity-crs" => "true",
               "modsecurity.conf" => modsecurity_config_content
             },
+            "extraContainers" => [
+              {
+                "name" => "modsecurity-log",
+                "image" => "busybox",
+                "args" => [
+                  "/bin/sh",
+                  "-c",
+                  "tail -f /var/log/modsec/audit.log"
+                ],
+                "volumeMounts" => [
+                  {
+                    "name" => "modsecurity-log-volume",
+                    "mountPath" => "/var/log/modsec",
+                    "readOnly" => true
+                  }
+                ],
+                "startupProbe" => {
+                  "exec" => {
+                    "command" => ["ls", "/var/log/modsec"]
+                  },
+                  "initialDelaySeconds" => MODSEC_SIDECAR_INITIAL_DELAY_SECONDS
+                }
+              }
+            ],
             "extraVolumeMounts" => [
               {
                 "name" => "modsecurity-template-volume",
                 "mountPath" => "/etc/nginx/modsecurity/modsecurity.conf",
                 "subPath" => "modsecurity.conf"
+              },
+              {
+                "name" => "modsecurity-log-volume",
+                "mountPath" => "/var/log/modsec"
               }
             ],
             "extraVolumes" => [
@@ -100,6 +129,10 @@ module Clusters
                     }
                   ]
                 }
+              },
+              {
+                "name" => "modsecurity-log-volume",
+                "emptyDir" => {}
               }
             ]
           }

app/models/concerns/noteable.rb

@@ -108,10 +108,6 @@ module Noteable
     discussions_resolvable? && resolvable_discussions.none?(&:to_be_resolved?)
   end
-  def discussions_to_be_resolved?
-    discussions_resolvable? && !discussions_resolved?
-  end
-
   def discussions_to_be_resolved
     @discussions_to_be_resolved ||= resolvable_discussions.select(&:to_be_resolved?)
   end

app/models/merge_request.rb

@@ -68,6 +68,7 @@ class MergeRequest < ApplicationRecord
   has_many :cached_closes_issues, through: :merge_requests_closing_issues, source: :issue
   has_many :pipelines_for_merge_request, foreign_key: 'merge_request_id', class_name: 'Ci::Pipeline'
   has_many :suggestions, through: :notes
+  has_many :unresolved_notes, -> { unresolved }, as: :noteable, class_name: 'Note'
   has_many :merge_request_assignees
   has_many :assignees, class_name: "User", through: :merge_request_assignees
@@ -211,7 +212,7 @@ class MergeRequest < ApplicationRecord
   scope :join_project, -> { joins(:target_project) }
   scope :references_project, -> { references(:target_project) }
   scope :with_api_entity_associations, -> {
-    preload(:assignees, :author, :notes, :labels, :milestone, :timelogs,
+    preload(:assignees, :author, :unresolved_notes, :labels, :milestone, :timelogs,
             latest_merge_request_diff: [:merge_request_diff_commits],
             metrics: [:latest_closed_by, :merged_by],
             target_project: [:route, { namespace: :route }],
@@ -923,7 +924,7 @@ class MergeRequest < ApplicationRecord
   def mergeable_discussions_state?
     return true unless project.only_allow_merge_if_all_discussions_are_resolved?
-    !discussions_to_be_resolved?
+    unresolved_notes.none?(&:to_be_resolved?)
   end
   def for_fork?

app/models/project.rb

@@ -92,6 +92,7 @@ class Project < ApplicationRecord
   default_value_for :snippets_enabled, gitlab_config_features.snippets
   default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
   default_value_for :remove_source_branch_after_merge, true
+  default_value_for(:ci_config_path) { Gitlab::CurrentSettings.default_ci_config_path }
   add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:projects_tokens_optional_encryption, default_enabled: true) ? :optional : :required }

app/views/admin/application_settings/_ci_cd.html.haml

@@ -53,5 +53,11 @@
           = s_('AdminSettings|Environment variables are protected by default')
       .form-text.text-muted
         = s_('AdminSettings|When creating a new environment variable it will be protected by default.')
+    .form-group
+      = f.label :ci_config_path, _('Default CI configuration path'), class: 'label-bold'
+      = f.text_field :default_ci_config_path, class: 'form-control', placeholder: '.gitlab-ci.yml'
+      %p.form-text.text-muted
+        = _("The default CI configuration path for new projects.").html_safe
+        = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'custom-ci-config-path'), target: '_blank'
   = f.submit _('Save changes'), class: "btn btn-success"

changelogs/unreleased/14707-add-modsec-logging-sidecar-to-ingress-controller.yml

+---
+title: Add modsecurity logging sidecar to ingress controller
+merge_request: 19600
+author:
+type: added

changelogs/unreleased/32935-preventing-accidental-project-deletion-index.yml

+---
+title: Add index on marked_for_deletion_at in projects table
+merge_request: 19788
+author:
+type: other

changelogs/unreleased/ab-projects-api-indexes-authenticated.yml

+---
+title: Add index for authenticated requests to projects API default endpoint
+merge_request: 19993
+author:
+type: performance

changelogs/unreleased/gitlab_ci_path.yml

+---
+title: Allow to define a default CI configuration path for new projects
+merge_request: 18073
+author: Mathieu Parent
+type: added

changelogs/unreleased/id-optimize-mergeable-discussions-state.yml

+---
+title: Optimize MergeRequest#mergeable_discussions_state? method
+merge_request: 19988
+author:
+type: performance

db/migrate/20191107220314_add_index_to_projects_on_marked_for_deletion.rb

+# frozen_string_literal: true
+
+class AddIndexToProjectsOnMarkedForDeletion < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, :marked_for_deletion_at, where: 'marked_for_deletion_at IS NOT NULL'
+  end
+
+  def down
+    remove_concurrent_index :projects, :marked_for_deletion_at
+  end
+end

db/migrate/20191111121500_default_ci_config_path.rb

+# frozen_string_literal: true
+
+class DefaultCiConfigPath < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def up
+    add_column :application_settings, :default_ci_config_path, :string, limit: 255
+  end
+
+  def down
+    remove_column :application_settings, :default_ci_config_path
+  end
+end

db/migrate/20191112221821_add_indexes_for_projects_api_default_params_authenticated.rb

+# frozen_string_literal: true
+
+class AddIndexesForProjectsApiDefaultParamsAuthenticated < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, %i(created_at id)
+    remove_concurrent_index_by_name :projects, 'index_projects_on_created_at'
+  end
+
+  def down
+    add_concurrent_index :projects, :created_at
+    remove_concurrent_index_by_name :projects, 'index_projects_on_created_at_and_id'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2019_11_12_214305) do
+ActiveRecord::Schema.define(version: 2019_11_12_221821) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -351,6 +351,7 @@ ActiveRecord::Schema.define(version: 2019_11_12_214305) do
     t.text "encrypted_eks_secret_access_key"
     t.string "snowplow_app_id"
     t.datetime_with_timezone "productivity_analytics_start_date"
+    t.string "default_ci_config_path", limit: 255
     t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id"
     t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id"
     t.index ["instance_administration_project_id"], name: "index_applicationsettings_on_instance_administration_project_id"
@@ -3121,7 +3122,7 @@ ActiveRecord::Schema.define(version: 2019_11_12_214305) do
     t.integer "marked_for_deletion_by_user_id"
     t.index "lower((name)::text)", name: "index_projects_on_lower_name"
     t.index ["archived", "pending_delete", "merge_requests_require_code_owner_approval"], name: "projects_requiring_code_owner_approval", where: "((pending_delete = false) AND (archived = false) AND (merge_requests_require_code_owner_approval = true))"
-    t.index ["created_at"], name: "index_projects_on_created_at"
+    t.index ["created_at", "id"], name: "index_projects_on_created_at_and_id"
     t.index ["creator_id"], name: "index_projects_on_creator_id"
     t.index ["description"], name: "index_projects_on_description_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["id", "repository_storage", "last_repository_updated_at"], name: "idx_projects_on_repository_storage_last_repository_updated_at"
@@ -3131,6 +3132,7 @@ ActiveRecord::Schema.define(version: 2019_11_12_214305) do
     t.index ["last_repository_check_at"], name: "index_projects_on_last_repository_check_at", where: "(last_repository_check_at IS NOT NULL)"
     t.index ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed"
     t.index ["last_repository_updated_at"], name: "index_projects_on_last_repository_updated_at"
+    t.index ["marked_for_deletion_at"], name: "index_projects_on_marked_for_deletion_at", where: "(marked_for_deletion_at IS NOT NULL)"
     t.index ["marked_for_deletion_by_user_id"], name: "index_projects_on_marked_for_deletion_by_user_id", where: "(marked_for_deletion_by_user_id IS NOT NULL)"
     t.index ["mirror_last_successful_update_at"], name: "index_projects_on_mirror_last_successful_update_at"
     t.index ["mirror_user_id"], name: "index_projects_on_mirror_user_id"

doc/administration/operations/sidekiq_memory_killer.md

@@ -34,7 +34,7 @@ The MemoryKiller is controlled using environment variables.
   In _daemon_ mode, the MemoryKiller checks the Sidekiq process RSS every 3 seconds
   (defined by `SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL`).
-- `SIDEKIQ_MEMORY_KILLER_MAX_RSS`: if this variable is set, and its value is greater
+- `SIDEKIQ_MEMORY_KILLER_MAX_RSS` (KB): if this variable is set, and its value is greater
   than 0, the MemoryKiller is enabled. Otherwise the MemoryKiller is disabled.
   `SIDEKIQ_MEMORY_KILLER_MAX_RSS` defines the Sidekiq process allowed RSS.
@@ -52,7 +52,7 @@ The MemoryKiller is controlled using environment variables.
   [in the Omnibus GitLab
   repository](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb).
-- `SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS`: is used by _daemon_ mode. If the Sidekiq
+- `SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS` (KB): is used by _daemon_ mode. If the Sidekiq
   process RSS (expressed in kilobytes) exceeds `SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS`,
   an immediate graceful restart of Sidekiq is triggered.

doc/api/settings.md

@@ -40,6 +40,7 @@ Example response:
    "domain_blacklist_enabled" : false,
    "domain_blacklist" : [],
    "created_at" : "2016-01-04T15:44:55.176Z",
+   "default_ci_config_path" : null,
    "default_project_visibility" : "private",
    "default_group_visibility" : "private",
    "gravatar_enabled" : true,
@@ -113,6 +114,7 @@ Example response:
   "restricted_visibility_levels": [],
   "max_attachment_size": 10,
   "session_expire_delay": 10080,
+  "default_ci_config_path" : null,
   "default_project_visibility": "internal",
   "default_snippet_visibility": "private",
   "default_group_visibility": "private",
@@ -198,6 +200,7 @@ are listed in the descriptions of the relevant settings.
 | `container_registry_token_expire_delay`  | integer          | no                                   | Container Registry token duration in minutes. |
 | `default_artifacts_expire_in`            | string           | no                                   | Set the default expiration time for each job's artifacts. |
 | `default_branch_protection`              | integer          | no                                   | Determine if developers can push to master. Can take: `0` _(not protected, both developers and maintainers can push new commits, force push, or delete the branch)_, `1` _(partially protected, developers and maintainers can push new commits, but cannot force push or delete the branch)_ or `2` _(fully protected, developers cannot push new commits, but maintainers can; no-one can force push or delete the branch)_ as a parameter. Default is `2`. |
+| `default_ci_config_path`                 | string           | no                                   | Default CI configuration path for new projects (`.gitlab-ci.yml` if not set). |
 | `default_group_visibility`               | string           | no                                   | What visibility level new groups receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. |
 | `default_project_creation`               | integer          | no                                   | Default project creation protection. Can take: `0` _(No one)_, `1` _(Maintainers)_ or `2` _(Developers + Maintainers)_|
 | `default_projects_limit`                 | integer          | no                                   | Project limit per user. Default is `100000`. |

doc/ci/yaml/README.md

@@ -2023,8 +2023,6 @@ Defining an empty array will skip downloading any artifacts for that job.
 The status of the previous job is not considered when using `dependencies`, so
 if it failed or it is a manual job that was not run, no error occurs.
----
-
 In the following example, we define two jobs with artifacts, `build:osx` and
 `build:linux`. When the `test:osx` is executed, the artifacts from `build:osx`
 will be downloaded and extracted in the context of the build. The same happens