Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/global.gitlab-ci.yml

@@ -102,7 +102,7 @@
     refs:
       - schedules
     variables:
-      - $REVIEW_APP_CLEANUP && $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
     kubernetes: active
 .use-pg:

.gitlab/ci/reports.gitlab-ci.yml

@@ -44,6 +44,8 @@ code_quality:
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.
 # See https://gitlab.com/gitlab-org/gitlab/issues/31371.
+# Once https://gitlab.com/gitlab-org/gitlab/merge_requests/16487 will be deployed
+# to GitLab.com, we should be able to use the template and set SAST_DISABLE_DIND: "true".
 sast:
   extends:
     - .default-retry

.gitlab/ci/review.gitlab-ci.yml

-.review-base:
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-only
-    - .only-review
-    - .only-code-qa-changes
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  dependencies: []
-  before_script:
-    - source scripts/utils.sh
-
 .review-docker:
   extends:
     - .default-tags
@@ -41,6 +29,25 @@ build-qa-image:
     - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY}
     - time docker push ${QA_IMAGE}
+schedule:review-cleanup:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-only
+    - .only-code-qa-changes
+    - .only-review-schedules
+  stage: prepare
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
+  allow_failure: true
+  environment:
+    name: review/auto-cleanup
+    action: stop
+  before_script:
+    - source scripts/utils.sh
+    - install_gitlab_gem
+  script:
+    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
+
 .review-build-cng-base:
   extends:
     - .default-only
@@ -68,9 +75,15 @@ schedule:review-build-cng:
   needs: ["gitlab:assets:compile"]
 .review-deploy-base:
-  extends: .review-base
-  allow_failure: true
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-only
+    - .only-code-qa-changes
   stage: review
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
+  dependencies: []
+  allow_failure: true
   variables:
     HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
     DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
@@ -105,6 +118,7 @@ schedule:review-build-cng:
 review-deploy:
   extends:
     - .review-deploy-base
+    - .only-review
   needs: ["review-build-cng"]
 schedule:review-deploy:
@@ -148,6 +162,7 @@ review-cleanup-failed-deployment:
     - .only-review
     - .only-code-qa-changes
   stage: qa
+  allow_failure: true
   variables:
     QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
     QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
@@ -176,13 +191,11 @@ review-cleanup-failed-deployment:
 review-qa-smoke:
   extends: .review-qa-base
-  allow_failure: true
   script:
     - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
 review-qa-all:
   extends: .review-qa-base
-  allow_failure: true
   when: manual
   parallel: 5
   script:
@@ -190,39 +203,32 @@ review-qa-all:
     - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
     - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation
-parallel-spec-reports:
+.review-performance-base:
   extends:
-    - .default-tags
-    - .default-only
+    - .review-docker
     - .only-code-qa-changes
-    - .only-review
-  image: ruby:2.6-alpine
-  stage: post-test
-  dependencies: ["review-qa-all"]
-  variables:
-    NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
-    BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"
+  stage: qa
   allow_failure: true
-  when: manual
+  before_script:
+    - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)"
+    - echo "${CI_ENVIRONMENT_URL}"
+    - mkdir -p gitlab-exporter
+    - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
+    - mkdir -p sitespeed-results
+  script:
+    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}"
+  after_script:
+    - mv sitespeed-results/data/performance.json performance.json
   artifacts:
-    when: always
     paths:
-      - qa/report-new.html
-      - qa/gitlab-qa-run-*
+      - sitespeed-results/
     reports:
-      junit: qa/gitlab-qa-run-*/**/rspec-*.xml
-  script:
-    - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/*
-    - gem install nokogiri --no-document
-    - cd qa/gitlab-qa-run-*/gitlab-*
-    - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_)
-    - cd -
-    - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}'
-    - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm
+      performance: performance.json
 review-performance:
-  extends: .review-qa-base
-  allow_failure: true
+  extends:
+    - .review-performance-base
+    - .only-review
   before_script:
     - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
@@ -241,24 +247,39 @@ review-performance:
 schedule:review-performance:
   extends:
-    - review-performance
+    - .review-performance-base
     - .only-review-schedules
   dependencies: ["schedule:review-deploy"]
-schedule:review-cleanup:
+parallel-spec-reports:
   extends:
-    - .review-base
-    - .only-review-schedules
-  stage: prepare
+    - .default-tags
+    - .default-only
+    - .only-code-qa-changes
+    - .only-review
+  image: ruby:2.6-alpine
+  stage: post-test
+  dependencies: ["review-qa-all"]
+  variables:
+    NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
+    BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"
   allow_failure: true
-  environment:
-    name: review/auto-cleanup
-    action: stop
-  before_script:
-    - source scripts/utils.sh
-    - install_gitlab_gem
+  when: manual
+  artifacts:
+    when: always
+    paths:
+      - qa/report-new.html
+      - qa/gitlab-qa-run-*
+    reports:
+      junit: qa/gitlab-qa-run-*/**/rspec-*.xml
   script:
-    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
+    - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/*
+    - gem install nokogiri --no-document
+    - cd qa/gitlab-qa-run-*/gitlab-*
+    - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_)
+    - cd -
+    - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}'
+    - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm
 danger-review:
   extends:

Gemfile

@@ -148,7 +148,7 @@ gem 'wikicloth', '0.8.1'
 gem 'asciidoctor', '~> 2.0.10'
 gem 'asciidoctor-include-ext', '~> 0.3.1', require: false
 gem 'asciidoctor-plantuml', '0.0.9'
-gem 'rouge', '~> 3.7'
+gem 'rouge', '~> 3.11.0'
 gem 'truncato', '~> 0.7.11'
 gem 'bootstrap_form', '~> 4.2.0'
 gem 'nokogiri', '~> 1.10.4'

Gemfile.lock

@@ -834,7 +834,7 @@ GEM
     retriable (3.1.2)
     rinku (2.0.0)
     rotp (2.1.2)
-    rouge (3.7.0)
+    rouge (3.11.0)
     rqrcode (0.7.0)
       chunky_png
     rqrcode-rails3 (0.1.7)
@@ -1276,7 +1276,7 @@ DEPENDENCIES
   redis-rails (~> 5.0.2)
   request_store (~> 1.3)
   responders (~> 2.0)
-  rouge (~> 3.7)
+  rouge (~> 3.11.0)
   rqrcode-rails3 (~> 0.1.7)
   rspec-parameterized
   rspec-rails (~> 3.8.0)

app/assets/javascripts/header.js

@@ -15,11 +15,10 @@ import { parseBoolean } from '~/lib/utils/common_utils';
  */
 export default function initTodoToggle() {
   $(document).on('todo:toggle', (e, count) => {
-    const parsedCount = parseInt(count, 10);
     const $todoPendingCount = $('.todos-count');
-    $todoPendingCount.text(highCountTrim(parsedCount));
-    $todoPendingCount.toggleClass('hidden', parsedCount === 0);
+    $todoPendingCount.text(highCountTrim(count));
+    $todoPendingCount.toggleClass('hidden', count === 0);
   });
 }

app/assets/javascripts/pages/dashboard/todos/index/todos.js

@@ -4,6 +4,7 @@ import $ from 'jquery';
 import { visitUrl } from '~/lib/utils/url_utility';
 import UsersSelect from '~/users_select';
 import { isMetaClick } from '~/lib/utils/common_utils';
+import { addDelimiter } from '~/lib/utils/text_utility';
 import { __ } from '~/locale';
 import flash from '~/flash';
 import axios from '~/lib/utils/axios_utils';
@@ -145,8 +146,8 @@ export default class Todos {
   updateBadges(data) {
     $(document).trigger('todo:toggle', data.count);
-    document.querySelector('.todos-pending .badge').innerHTML = data.count;
-    document.querySelector('.todos-done .badge').innerHTML = data.done_count;
+    document.querySelector('.todos-pending .badge').innerHTML = addDelimiter(data.count);
+    document.querySelector('.todos-done .badge').innerHTML = addDelimiter(data.done_count);
   }
   goToTodoUrl(e) {

app/controllers/dashboard/todos_controller.rb

@@ -78,8 +78,8 @@ class Dashboard::TodosController < Dashboard::ApplicationController
   def todos_counts
     {
-      count: number_with_delimiter(current_user.todos_pending_count),
-      done_count: number_with_delimiter(current_user.todos_done_count)
+      count: current_user.todos_pending_count,
+      done_count: current_user.todos_done_count
     }
   end

app/views/projects/issues/_issue.html.haml

@@ -43,7 +43,10 @@
       .issuable-meta
         %ul.controls
-          - if issue.closed?
+          - if issue.moved?
+            %li.issuable-status
+              = _('CLOSED (MOVED)')
+          - elsif issue.closed?
             %li.issuable-status
               = _('CLOSED')
           - if issue.assignees.any?

changelogs/unreleased/66123-over-a-thousand-todos-displays-count-of-one.yml

+---
+title: Fix for count in todo badge when user has over 1,000 todos. Will now correctly
+  display todo count after user marks some todos as done.
+merge_request: 16844
+author: Jesse Hall @jessehall3
+type: fixed

changelogs/unreleased/66330-confidential-quick-action-is-available-for-confidential-issues.yml

+---
+title: Changed confidential quick action to only be available on non confidential issues
+merge_request: 16902
+author: Marc Schwede
+type: fixed

changelogs/unreleased/sh-bump-rouge-3-11-0.yml

+---
+title: Upgrade Rouge to v3.11.0
+merge_request: 17011
+author:
+type: other

changelogs/unreleased/winh-issue-list-moved.yml

+---
+title: Display if an issue was moved in issue list
+merge_request: 17102
+author:
+type: changed

config/initializers/sidekiq.rb

@@ -28,16 +28,18 @@ if Rails.env.development?
 end
 enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
-enable_sidekiq_monitor = ENV.fetch("SIDEKIQ_MONITOR_WORKER", 0).to_i.nonzero?
+enable_sidekiq_memory_killer = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero?
+use_sidekiq_daemon_memory_killer = ENV["SIDEKIQ_DAEMON_MEMORY_KILLER"].to_i.nonzero?
+use_sidekiq_legacy_memory_killer = !use_sidekiq_daemon_memory_killer
 Sidekiq.configure_server do |config|
   config.redis = queues_config_hash
   config.server_middleware do |chain|
-    chain.add Gitlab::SidekiqMiddleware::Monitor if enable_sidekiq_monitor
+    chain.add Gitlab::SidekiqMiddleware::Monitor
     chain.add Gitlab::SidekiqMiddleware::Metrics if Settings.monitoring.sidekiq_exporter
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] && !enable_json_logs
-    chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS']
+    chain.add Gitlab::SidekiqMiddleware::MemoryKiller if enable_sidekiq_memory_killer && use_sidekiq_legacy_memory_killer
     chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0'
     chain.add Gitlab::SidekiqMiddleware::BatchLoader
     chain.add Gitlab::SidekiqMiddleware::CorrelationLogger
@@ -60,7 +62,11 @@ Sidekiq.configure_server do |config|
     # Sidekiq (e.g. in an initializer).
     ActiveRecord::Base.clear_all_connections!
-    Gitlab::SidekiqDaemon::Monitor.instance.start if enable_sidekiq_monitor
+    # Start monitor to track running jobs. By default, cancel job is not enabled
+    # To cancel job, it requires `SIDEKIQ_MONITOR_WORKER=1` to enable notification channel
+    Gitlab::SidekiqDaemon::Monitor.instance.start
+
+    Gitlab::SidekiqDaemon::MemoryKiller.instance.start if enable_sidekiq_memory_killer && use_sidekiq_daemon_memory_killer
   end
   if enable_reliable_fetch?

db/migrate/20190903150358_create_analytics_repository_files_table.rb

+# frozen_string_literal: true
+
+class CreateAnalyticsRepositoryFilesTable < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :analytics_repository_files do |t|
+      t.references :project,
+        index: false,
+        foreign_key: { on_delete: :cascade },
+        null: false
+      t.string :file_path,
+        limit: 4096,
+        null: false
+    end
+
+    add_index :analytics_repository_files, [:project_id, :file_path], unique: true
+  end
+end

db/migrate/20190903150435_create_analytics_repository_file_edits_table.rb

+# frozen_string_literal: true
+
+class CreateAnalyticsRepositoryFileEditsTable < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :analytics_repository_file_edits do |t|
+      t.references :project,
+        index: true,
+        foreign_key: { on_delete: :cascade }, null: false
+      t.references :analytics_repository_file,
+        index: false,
+        foreign_key: { on_delete: :cascade },
+        null: false
+      t.date :committed_date,
+        null: false
+      t.integer :num_edits,
+        null: false,
+        default: 0
+    end
+
+    add_index :analytics_repository_file_edits,
+      [:analytics_repository_file_id, :committed_date, :project_id],
+      name: 'index_file_edits_on_committed_date_file_id_and_project_id',
+      unique: true
+  end
+end

db/schema.rb

@@ -93,6 +93,21 @@ ActiveRecord::Schema.define(version: 2019_09_18_104222) do
     t.index ["project_id"], name: "analytics_repository_languages_on_project_id"
   end
+  create_table "analytics_repository_file_edits", force: :cascade do |t|
+    t.bigint "project_id", null: false
+    t.bigint "analytics_repository_file_id", null: false
+    t.date "committed_date", null: false
+    t.integer "num_edits", default: 0, null: false
+    t.index ["analytics_repository_file_id", "committed_date", "project_id"], name: "index_file_edits_on_committed_date_file_id_and_project_id", unique: true
+    t.index ["project_id"], name: "index_analytics_repository_file_edits_on_project_id"
+  end
+
+  create_table "analytics_repository_files", force: :cascade do |t|
+    t.bigint "project_id", null: false
+    t.string "file_path", limit: 4096, null: false
+    t.index ["project_id", "file_path"], name: "index_analytics_repository_files_on_project_id_and_file_path", unique: true
+  end
+
   create_table "appearances", id: :serial, force: :cascade do |t|
     t.string "title", null: false
     t.text "description", null: false
@@ -3789,6 +3804,9 @@ ActiveRecord::Schema.define(version: 2019_09_18_104222) do
   add_foreign_key "analytics_cycle_analytics_project_stages", "projects", on_delete: :cascade
   add_foreign_key "analytics_language_trend_repository_languages", "programming_languages", on_delete: :cascade
   add_foreign_key "analytics_language_trend_repository_languages", "projects", on_delete: :cascade
+  add_foreign_key "analytics_repository_file_edits", "analytics_repository_files", on_delete: :cascade
+  add_foreign_key "analytics_repository_file_edits", "projects", on_delete: :cascade
+  add_foreign_key "analytics_repository_files", "projects", on_delete: :cascade
   add_foreign_key "application_settings", "namespaces", column: "custom_project_templates_group_id", on_delete: :nullify
   add_foreign_key "application_settings", "projects", column: "file_template_project_id", name: "fk_ec757bd087", on_delete: :nullify
   add_foreign_key "application_settings", "projects", column: "instance_administration_project_id", on_delete: :nullify

doc/administration/geo/replication/configuration.md

@@ -25,7 +25,7 @@ Any change that requires access to the **Admin Area** needs to be done in the
 GitLab stores a number of secret values in the `/etc/gitlab/gitlab-secrets.json`
 file which *must* be the same on all nodes. Until there is
-a means of automatically replicating these between nodes (see issue [gitlab-org/gitlab-ee#3789]),
+a means of automatically replicating these between nodes (see issue [gitlab-org/gitlab#3789]),
 they must be manually replicated to the **secondary** node.
 1. SSH into the **primary** node, and execute the command below:
@@ -299,7 +299,7 @@ See the [troubleshooting document](troubleshooting.md).
 [setup-geo-omnibus]: index.md#using-omnibus-gitlab
 [Hashed Storage]: ../../repository_storage_types.md
 [Disaster Recovery]: ../disaster_recovery/index.md
-[gitlab-org/gitlab-ee#3789]: https://gitlab.com/gitlab-org/gitlab/issues/3789
+[gitlab-org/gitlab#3789]: https://gitlab.com/gitlab-org/gitlab/issues/3789
 [gitlab-com/infrastructure#2821]: https://gitlab.com/gitlab-com/infrastructure/issues/2821
 [omnibus-ssl]: https://docs.gitlab.com/omnibus/settings/ssl.html
 [using-geo]: using_a_geo_server.md

doc/administration/high_availability/README.md

@@ -165,7 +165,7 @@ contention due to certain workloads.
 #### Reference Architecture
 - **Supported Users (approximate):** 10,000
-- **Known Issues:** While validating the reference architecture, slow endpoints were discovered and are being investigated. [gitlab-org/gitlab-ce/issues/64335](https://gitlab.com/gitlab-org/gitlab-foss/issues/64335)
+- **Known Issues:** While validating the reference architecture, slow endpoints were discovered and are being investigated. [gitlab-org/gitlab-foss/issues/64335](https://gitlab.com/gitlab-org/gitlab-foss/issues/64335)
 The Support and Quality teams built, performance tested, and validated an
 environment that supports about 10,000 users. The specifications below are a

doc/administration/high_availability/database.md

@@ -1049,7 +1049,7 @@ If you're running into an issue with a component not outlined here, be sure to c
 ## Configure using Omnibus
 **Note**: We recommend that you follow the instructions here for a full [PostgreSQL cluster](#high-availability-with-gitlab-omnibus-premium-only).
-If you are reading this section due to an old bookmark, you can find that old documentation [in the repository](https://gitlab.com/gitlab-org/gitlab-foss/blob/v10.1.4/doc/administration/high_availability/database.md#configure-using-omnibus).
+If you are reading this section due to an old bookmark, you can find that old documentation [in the repository](https://gitlab.com/gitlab-org/gitlab/blob/v10.1.4/doc/administration/high_availability/database.md#configure-using-omnibus).
 Read more on high-availability configuration: