Add latest changes from gitlab-org/gitlab@master

.gitlab-ci.yml

@@ -30,6 +30,7 @@ after_script:
   - date
 include:
+  - local: .gitlab/ci/cache-repo.gitlab-ci.yml
   - local: .gitlab/ci/cng.gitlab-ci.yml
   - local: .gitlab/ci/docs.gitlab-ci.yml
   - local: .gitlab/ci/frontend.gitlab-ci.yml

.gitlab/ci/cache-repo.gitlab-ci.yml

+# Builds a cached .tar.gz of the master branch with full history and
+# uploads it to Google Cloud Storage. This archive is downloaded by a
+# script defined by a CI/CD variable named CI_PRE_CLONE_SCRIPT. This has
+# two benefits:
+#
+# 1. It speeds up builds. A 800 MB download only takes seconds.
+# 2. It significantly reduces load on the file server. Smaller deltas
+#    means less time spent in git pack-objects.
+#
+# Since the destination directory of the archive depends on the project
+# ID, this is only run on GitLab.com.
+#
+# CI_REPO_CACHE_CREDENTIALS contains the Google Cloud service account
+# JSON for uploading to the gitlab-ci-git-repo-cache bucket. These
+# credentials are stored in the Production vault.
+#
+# Note that this bucket should be located in the same continent as the
+# runner, or network egress charges will apply:
+# https://cloud.google.com/storage/pricing
+cache-repo:
+  extends:
+    - .only:variables_refs-canonical-dot-com-schedules
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+  stage: sync
+  allow_failure: true
+  variables:
+    GIT_DEPTH: 0
+    TAR_FILENAME: /tmp/gitlab-master.tar
+  script:
+    - gcloud auth activate-service-account --key-file=$CI_REPO_CACHE_CREDENTIALS
+    - tar cf $TAR_FILENAME .
+    - gzip $TAR_FILENAME
+    - gsutil cp $TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/gitlab-master.tar.gz

.gitlab/ci/reports.gitlab-ci.yml

@@ -37,6 +37,8 @@ code_quality:
   artifacts:
     reports:
       codequality: gl-code-quality-report.json
+    paths:
+      - gl-code-quality-report.json
     expire_in: 1 week
   dependencies: []
   except:

app/policies/project_snippet_policy.rb

@@ -38,6 +38,10 @@ class ProjectSnippetPolicy < BasePolicy
   rule { public_snippet }.enable :read_project_snippet
+  rule { is_author & ~project.reporter & ~admin }.policy do
+    prevent :admin_project_snippet
+  end
+
   rule { is_author | admin }.policy do
     enable :read_project_snippet
     enable :update_project_snippet

app/views/projects/snippets/_actions.html.haml

@@ -4,7 +4,7 @@
   - if can?(current_user, :update_project_snippet, @snippet)
     = link_to edit_project_snippet_path(@project, @snippet), class: "btn btn-grouped" do
       = _('Edit')
-  - if can?(current_user, :update_project_snippet, @snippet)
+  - if can?(current_user, :admin_project_snippet, @snippet)
     = link_to project_snippet_path(@project, @snippet), method: :delete, data: { confirm: _("Are you sure?") }, class: "btn btn-grouped btn-inverted btn-remove", title: _('Delete Snippet') do
       = _('Delete')
   - if can?(current_user, :create_project_snippet, @project)
@@ -23,7 +23,7 @@
           %li
             = link_to new_project_snippet_path(@project), title: _("New snippet") do
               = _('New snippet')
-        - if can?(current_user, :update_project_snippet, @snippet)
+        - if can?(current_user, :admin_project_snippet, @snippet)
           %li
             = link_to project_snippet_path(@project, @snippet), method: :delete, data: { confirm: _("Are you sure?") }, title: _('Delete Snippet') do
               = _('Delete')

changelogs/unreleased/33596-package-ci-build-data.yml

+---
+title: Add Pipeline Metadata to Packages
+merge_request: 19796
+author:
+type: added

changelogs/unreleased/37387-do-not-clean-sidekiq-metrics.yml

+---
+title: Do not clean the prometheus metrics directory for sidekiq
+merge_request: 21671
+author:
+type: fixed

changelogs/unreleased/al-31836-guest-access-to-delete-snippets-bug.yml

+---
+title: Guest users should not delete project snippets they created
+merge_request: 20477
+author:
+type: fixed

config/initializers/7_prometheus_metrics.rb

@@ -32,15 +32,8 @@ end
 Sidekiq.configure_server do |config|
   config.on(:startup) do
-    # webserver metrics are cleaned up in config.ru: `warmup` block
-    Prometheus::CleanupMultiprocDirService.new.execute
-    # In production, sidekiq is run in a multi-process setup where processes might interfere
-    # with each other cleaning up and reinitializing prometheus database files, which is why
-    # we're re-doing the work every time here.
-    # A cleaner solution would be to run the cleanup pre-fork, and the initialization once
-    # after all workers have forked, but I don't know how at this point.
-    ::Prometheus::Client.reinitialize_on_pid_change(force: true)
-
+    # Do not clean the metrics directory here - the supervisor script should
+    # have already taken care of that
     Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
   end
 end

config/initializers/devise_dynamic_password_length_validation.rb

@@ -22,6 +22,8 @@ end
 # This initializer can be removed as soon as https://github.com/plataformatec/devise/pull/5166
 # is merged into Devise.
+
+# TODO: Update Devise. Issue: https://gitlab.com/gitlab-org/gitlab/issues/118450
 if length_validator_supports_dynamic_length_checks?(password_length_validator)
   raise "Devise now supports dynamic length checks, please remove the monkey patch in #{__FILE__}"
 else

db/migrate/20191108031900_create_package_build_info.rb

+# frozen_string_literal: true
+
+class CreatePackageBuildInfo < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    create_table :packages_build_infos do |t|
+      t.references :package, null: false, foreign_key: { to_table: :packages_packages, on_delete: :cascade }, type: :integer, index: { unique: true }
+      t.references :pipeline, index: true, null: true, foreign_key: { to_table: :ci_pipelines, on_delete: :nullify }, type: :integer
+    end
+  end
+end

db/schema.rb

@@ -2846,6 +2846,13 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do
     t.index ["project_id", "token_encrypted"], name: "index_feature_flags_clients_on_project_id_and_token_encrypted", unique: true
   end
+  create_table "packages_build_infos", force: :cascade do |t|
+    t.integer "package_id", null: false
+    t.integer "pipeline_id"
+    t.index ["package_id"], name: "index_packages_build_infos_on_package_id", unique: true
+    t.index ["pipeline_id"], name: "index_packages_build_infos_on_pipeline_id"
+  end
+
   create_table "packages_conan_file_metadata", force: :cascade do |t|
     t.bigint "package_file_id", null: false
     t.datetime_with_timezone "created_at", null: false
@@ -4644,6 +4651,8 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do
   add_foreign_key "operations_feature_flag_scopes", "operations_feature_flags", column: "feature_flag_id", on_delete: :cascade
   add_foreign_key "operations_feature_flags", "projects", on_delete: :cascade
   add_foreign_key "operations_feature_flags_clients", "projects", on_delete: :cascade
+  add_foreign_key "packages_build_infos", "ci_pipelines", column: "pipeline_id", on_delete: :nullify
+  add_foreign_key "packages_build_infos", "packages_packages", column: "package_id", on_delete: :cascade
   add_foreign_key "packages_conan_file_metadata", "packages_package_files", column: "package_file_id", on_delete: :cascade
   add_foreign_key "packages_conan_metadata", "packages_packages", column: "package_id", on_delete: :cascade
   add_foreign_key "packages_dependency_links", "packages_dependencies", column: "dependency_id", on_delete: :cascade

doc/administration/pages/index.md

@@ -305,7 +305,7 @@ Pages access control is disabled by default. To enable it:
    ```
 1. [Reconfigure GitLab][reconfigure].
-1. Users can now configure it in their [projects' settings](../../user/project/pages/introduction.md#gitlab-pages-access-control-core).
+1. Users can now configure it in their [projects' settings](../../user/project/pages/pages_access_control.md).
 ### Running behind a proxy
@@ -500,6 +500,7 @@ then you must use the following procedure to configure [access control](#access-
    ```ruby
    gitlab_pages['gitlab_server'] = "https://<your-gitlab-server-URL>"
+   gitlab_pages['access_control'] = true
    ```
 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.

doc/ci/examples/test-and-deploy-python-application-to-heroku.md

@@ -83,7 +83,7 @@ gitlab-runner register \
   --description "python-3.5" \
   --executor "docker" \
   --docker-image python:3.5 \
-  --docker-postgres latest
+  --docker-services postgres:latest
 ```
 With the command above, you create a runner that uses the [`python:3.5`](https://hub.docker.com/_/python) image and uses a [PostgreSQL](https://hub.docker.com/_/postgres) database.

doc/development/changelog.md

@@ -33,7 +33,8 @@ the `author` field. GitLab team members **should not**.
 ## What warrants a changelog entry?
-- Any change that introduces a database migration **must** have a changelog entry.
+- Any change that introduces a database migration, whether it's regular, post,
+  or data migration, **must** have a changelog entry.
 - Any user-facing change **should** have a changelog entry. Example: "GitLab now
   uses system fonts for all text."
 - Performance improvements **should** have a changelog entry.
@@ -43,8 +44,8 @@ the `author` field. GitLab team members **should not**.
 - Any docs-only changes **should not** have a changelog entry.
 - Any change behind a feature flag **should not** have a changelog entry. The
   entry should be added [in the merge request removing the feature flags](feature_flags/development.md).
-  If the change includes a database migration, there should be a changelog entry
-  for the migration change.
+  If the change includes a database migration (regular, post, or data migration),
+  there should be a changelog entry for the migration change.
 - A fix for a regression introduced and then fixed in the same release (i.e.,
   fixing a bug introduced during a monthly release candidate) **should not**
   have a changelog entry.

doc/user/project/pages/pages_access_control.md

@@ -11,7 +11,7 @@ You can enable Pages access control on your project, so that only
 [members of your project](../../permissions.md#project-members-permissions)
 (at least Guest) can access your website:
-1. Navigate to your project's **Settings > General > Permissions**.
+1. Navigate to your project's **Settings > General** and expand **Visibility, project features, permissions**.
 1. Toggle the **Pages** button to enable the access control.
    NOTE: **Note:**

lib/tasks/gitlab/import_export/import.rake

@@ -64,16 +64,25 @@ class GitlabProjectImport
   private
+  def with_request_store
+    RequestStore.begin!
+    yield
+  ensure
+    RequestStore.end!
+    RequestStore.clear!
+  end
+
   # We want to ensure that all Sidekiq jobs are executed
   # synchronously as part of that process.
   # This ensures that all expensive operations do not escape
   # to general Sidekiq clusters/nodes.
   def run_isolated_sidekiq_job
     Sidekiq::Testing.fake! do
-      @project = create_project
-
-      execute_sidekiq_job
+      with_request_store do
+        @project = create_project
+        execute_sidekiq_job
+      end
       true
     end
   end

locale/gitlab.pot

@@ -3226,6 +3226,18 @@ msgstr ""
 msgid "Checkout"
 msgstr ""
+msgid "Checkout|1. Your profile"
+msgstr ""
+
+msgid "Checkout|2. Checkout"
+msgstr ""
+
+msgid "Checkout|3. Your GitLab group"
+msgstr ""
+
+msgid "Checkout|Checkout"
+msgstr ""
+
 msgid "Cherry-pick this commit"
 msgstr ""

spec/policies/project_snippet_policy_spec.rb

@@ -4,10 +4,12 @@ require 'spec_helper'
 # Snippet visibility scenarios are included in more details in spec/support/snippet_visibility.rb
 describe ProjectSnippetPolicy do
-  let(:regular_user) { create(:user) }
-  let(:external_user) { create(:user, :external) }
-  let(:project) { create(:project, :public) }
-  let(:snippet) { create(:project_snippet, snippet_visibility, project: project) }
+  let_it_be(:regular_user) { create(:user) }
+  let_it_be(:other_user) { create(:user) }
+  let_it_be(:external_user) { create(:user, :external) }
+  let_it_be(:project) { create(:project, :public) }
+  let(:snippet) { create(:project_snippet, snippet_visibility, project: project, author: author) }
+  let(:author) { other_user }
   let(:author_permissions) do
     [
       :update_project_snippet,
@@ -17,6 +19,65 @@ describe ProjectSnippetPolicy do
   subject { described_class.new(current_user, snippet) }
+  shared_examples 'regular user access rights' do
+    context 'project team member (non guest)' do
+      before do
+        project.add_developer(current_user)
+      end
+
+      it do
+        expect_allowed(:read_project_snippet, :create_note)
+        expect_disallowed(*author_permissions)
+      end
+    end
+
+    context 'project team member (guest)' do
+      before do
+        project.add_guest(current_user)
+      end
+
+      context 'not snippet author' do
+        it do
+          expect_allowed(:read_project_snippet, :create_note)
+          expect_disallowed(:admin_project_snippet)
+        end
+      end
+    end
+
+    context 'snippet author' do
+      let(:author) { current_user }
+
+      context 'project member (non guest)' do
+        before do
+          project.add_developer(current_user)
+        end
+
+        it do
+          expect_allowed(:read_project_snippet, :create_note)
+          expect_allowed(*author_permissions)
+        end
+      end
+
+      context 'project member (guest)' do
+        before do
+          project.add_guest(current_user)
+        end
+
+        it do
+          expect_allowed(:read_project_snippet, :create_note)
+          expect_disallowed(:admin_project_snippet)
+        end
+      end
+
+      context 'not a project member' do
+        it do
+          expect_allowed(:read_project_snippet, :create_note)
+          expect_disallowed(:admin_project_snippet)
+        end
+      end
+    end
+  end
+
   context 'public snippet' do
     let(:snippet_visibility) { :public }
@@ -36,6 +97,8 @@ describe ProjectSnippetPolicy do
         expect_allowed(:read_project_snippet, :create_note)
         expect_disallowed(*author_permissions)
       end
+
+      it_behaves_like 'regular user access rights'
     end
     context 'external user' do
@@ -45,6 +108,17 @@ describe ProjectSnippetPolicy do
         expect_allowed(:read_project_snippet, :create_note)
         expect_disallowed(*author_permissions)
       end
+
+      context 'project team member' do
+        before do
+          project.add_developer(external_user)
+        end
+
+        it do
+          expect_allowed(:read_project_snippet, :create_note)
+          expect_disallowed(*author_permissions)
+        end
+      end
     end
   end
@@ -67,6 +141,8 @@ describe ProjectSnippetPolicy do
         expect_allowed(:read_project_snippet, :create_note)
         expect_disallowed(*author_permissions)
       end
+
+      it_behaves_like 'regular user access rights'
     end
     context 'external user' do
@@ -110,33 +186,20 @@ describe ProjectSnippetPolicy do
         expect_disallowed(*author_permissions)
       end
-      context 'snippet author' do
-        let(:snippet) { create(:project_snippet, :private, author: regular_user, project: project) }
-
-        it do
-          expect_allowed(:read_project_snippet, :create_note)
-          expect_allowed(*author_permissions)
-        end
-      end
-      context 'project team member normal user' do
-        before do
-          project.add_developer(regular_user)
-        end
-        it do
-          expect_allowed(:read_project_snippet, :create_note)
-          expect_disallowed(*author_permissions)
-        end
+      it_behaves_like 'regular user access rights'
+    end
+    context 'external user' do
+      let(:current_user) { external_user }
+      it do
+        expect_disallowed(:read_project_snippet, :create_note)
+        expect_disallowed(*author_permissions)
       end
-    end
-    context 'external user' do
       context 'project team member' do
-        let(:current_user) { external_user }
-
         before do
-          project.add_developer(external_user)
+          project.add_developer(current_user)
         end
         it do