Add latest changes from gitlab-org/gitlab@master

app/models/snippet.rb

@@ -283,6 +283,10 @@ class Snippet < ApplicationRecord
     end
   end
+  def url_to_repo
+    Gitlab::Shell.url_to_repo(full_path.delete('@'))
+  end
+
   def repository_storage
     snippet_repository&.shard_name ||
       Gitlab::CurrentSettings.pick_repository_storage

app/views/layouts/nav/sidebar/_project.html.haml

@@ -159,6 +159,8 @@
                 %span.badge.badge-pill.count.merge_counter.js-merge-counter.fly-out-badge
                   = number_with_delimiter(@project.open_merge_requests_count)
+      = render_if_exists "layouts/nav/requirements_link", project: @project
+
       - if project_nav_tab? :pipelines
         = nav_link(controller: [:pipelines, :builds, :jobs, :pipeline_schedules, :artifacts], unless: -> { current_path?('projects/pipelines#charts') }) do
           = link_to project_pipelines_path(@project), class: 'shortcuts-pipelines qa-link-pipelines rspec-link-pipelines', data: { qa_selector: 'ci_cd_link' } do

app/workers/concerns/application_worker.rb

@@ -13,6 +13,17 @@ module ApplicationWorker
   included do
     set_queue
+
+    def structured_payload(payload = {})
+      context = Labkit::Context.current.to_h.merge(
+        'class' => self.class,
+        'job_status' => 'running',
+        'queue' => self.class.queue,
+        'jid' => jid
+      )
+
+      payload.stringify_keys.merge(context)
+    end
   end
   class_methods do

changelogs/unreleased/fj-205646-fix-project-moved-message.yml

+---
+title: Fix project moved message after git operation
+merge_request: 27341
+author:
+type: fixed

changelogs/unreleased/fj-fix-snippet-url-to-repo.yml

+---
+title: Fix remove special chars from snippet url_to_repo
+merge_request: 27390
+author:
+type: fixed

config/sidekiq_queues.yml

@@ -232,6 +232,8 @@
   - 2
 - - service_desk_email_receiver
   - 1
+- - status_page_publish_incident
+  - 1
 - - sync_seat_link_request
   - 1
 - - system_hook_push

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -182,7 +182,7 @@ Unicorn specific metrics, when Unicorn is used.
 | `unicorn_queued_connections` | Gauge | 11.0  | The number of queued Unicorn connections           |
 | `unicorn_workers`            | Gauge | 12.0  | The number of Unicorn workers                      |
-## Puma Metrics **(EXPERIMENTAL)**
+## Puma Metrics
 When Puma is used instead of Unicorn, the following metrics are available:

doc/administration/operations/puma.md

@@ -2,8 +2,8 @@
 ## Puma
-GitLab plans to use [Puma](https://github.com/puma/puma) to replace
-[Unicorn](https://bogomips.org/unicorn/).
+As of GitLab 12.9, [Puma](https://github.com/puma/puma) has replaced [Unicorn](https://bogomips.org/unicorn/).
+as the default web server.
 ## Why switch to Puma?

doc/administration/packages/dependency_proxy.md

@@ -11,9 +11,8 @@ dependency proxies, see the [user guide](../../user/group/dependency_proxy/index
 NOTE: **Note:**
 Dependency proxy requires the Puma web server to be enabled.
-Puma support is EXPERIMENTAL at this time.
-To enable the Dependency proxy feature:
+To enable the dependency proxy feature:
 **Omnibus GitLab installations**
@@ -37,7 +36,9 @@ To enable the Dependency proxy feature:
    ```
 1. [Restart GitLab](../restart_gitlab.md#installations-from-source "How to restart GitLab") for the changes to take effect.
-1. Enable the [Puma web server](../../install/installation.md#using-puma).
+
+Since Puma is already the default web server for installations from source as of GitLab 12.9,
+no further changes are needed.
 ## Changing the storage path

doc/administration/pages/index.md

@@ -566,6 +566,51 @@ GitLab Pages are part of the [regular backup][backup], so there is no separate b
 You should strongly consider running GitLab Pages under a different hostname
 than GitLab to prevent XSS attacks.
+<!-- ## Troubleshooting
+
+Include any troubleshooting steps that you can foresee. If you know beforehand what issues
+one might have when setting this up, or when something is changed, or on upgrading, it's
+important to describe those, too. Think of things that may go wrong and include them here.
+This is important to minimize requests for support, and to avoid doc comments with
+questions that you know someone might ask.
+
+Each scenario can be a third-level heading, e.g. `### Getting error message X`.
+If you have none to add when creating a doc, leave this section in place
+but commented out to help encourage others to add to it in the future. -->
+
+## Troubleshooting
+
+### `open /etc/ssl/ca-bundle.pem: permission denied`
+
+GitLab Pages runs inside a `chroot` jail, usually in a uniquely numbered directory like
+`/tmp/gitlab-pages-*`.
+
+Within the jail, a bundle of trusted certificates is
+provided at `/etc/ssl/ca-bundle.pem`. It's
+[copied there](https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/51)
+from `/opt/gitlab/embedded/ssl/certs/cacert.pem`
+as part of starting up Pages.
+
+If the permissions on the source file are incorrect (they should be `0644`) then
+the file inside the `chroot` jail will also be wrong.
+
+Pages will log errors in `/var/log/gitlab/gitlab-pages/current` like:
+
+```plaintext
+x509: failed to load system roots and no roots provided
+open /etc/ssl/ca-bundle.pem: permission denied
+```
+
+The use of a `chroot` jail makes this error misleading, as it is not
+referring to `/etc/ssl` on the root filesystem.
+
+The fix is to correct the source file permissions and restart Pages:
+
+```shell
+sudo chmod 644 /opt/gitlab/embedded/ssl/certs/cacert.pem
+sudo gitlab-ctl restart gitlab-pages
+```
+
 [backup]: ../../raketasks/backup_restore.md
 [ce-14605]: https://gitlab.com/gitlab-org/gitlab-foss/issues/14605
 [ee-80]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80

doc/development/README.md

@@ -163,6 +163,7 @@ Complementary reads:
 - [Jira Connect app](integrations/jira_connect.md)
 - [Security Scanners](integrations/secure.md)
+- [Secure Partner Integration](integrations/secure_partner_integration.md)
 ## Testing guides

doc/development/integrations/secure_partner_integration.md

+# Secure Partner Integration - Onboarding Process
+
+If you want to integrate your product with the [Secure Stage](https://about.gitlab.com/direction/secure),
+this page will help you understand the developer workflow GitLab intends for
+our users to follow with regards to security results. These should be used as
+guidelines so you can build an integration that fits with the workflow GitLab
+users are already familiar with.
+
+This page also provides resources for the technical work associated
+with [onboarding as a partner](https://about.gitlab.com/partners/integrate/).
+The steps below are a high-level view of what needs to be done to complete an
+integration as well as linking to more detailed resources for how to do so.
+
+## What is the GitLab Developer Workflow?
+
+This workflow is how GitLab users interact with our product and expect it to
+function. Understanding how users use GitLab today will help you choose the
+best place to integrate your own product and its results into GitLab.
+
+- Developers want to write code without using a new tool to consume results
+  or address feedback about the item they are working on. Staying inside a
+  single tool, GitLab, helps them to stay focused on finishing the code and
+  projects they are working on.
+- Developers commit code to a Git branch. The developer creates a merge request (MR)
+  inside GitLab where these changes can be reviewed. The MR triggers a GitLab
+  pipeline to run associated jobs, including security checks, on the code.
+- Pipeline jobs serve a variety of purposes. Jobs can do scanning for and have
+  implications for app security, corporate policy, or compliance. When complete,
+  the job reports back on its status and creates a
+  [job artifact](../../user/project/pipelines/job_artifacts.md) as a result.
+- The [Merge Request Security Widget](../../user/project/merge_requests/index.md#security-reports-ultimate)
+  displays the results of the pipeline's security checks and the developer can
+  review them. The developer can review both a summary and a detailed version
+  of the results.
+- If certain policies (such as [merge request approvals](../../user/project/merge_requests/merge_request_approvals.md))
+  are in place for a project, developers must resolve specific findings or get
+  an approval from a specific list of people.
+- The [security dashboard](../../user/application_security/security_dashboard/index.md#gitlab-security-dashboard-ultimate)
+  also shows results which can developers can use to quickly see all the
+  vulnerabilities that need to be addressed in the code.
+- When the developer reads the details about a vulnerability, they are
+  presented with additional information and choices on next steps:
+    1. Create Issue (Confirm finding): Creates a new issue to be prioritized.
+    1. Add Comment and Dismiss Vulnerability: When dismissing a finding, users
+       can comment to note items that they
+       have mitigated, that they accept the vulnerability, or that the
+       vulnerability is a false positive.
+    1. Auto-Remediation / Create Merge Request: A fix for the vulnerability can
+       be offered, allowing an easy solution that does not require extra effort
+       from users. This should be offered whenever possible.
+    1. Links: Vulnerabilities can link out external sites or sources for users
+       to get more data around the vulnerability.
+
+## How to onboard
+
+This section describes the steps you need to complete to onboard as a partner
+and complete an intgration with the Secure stage.
+
+1. Read about our [partnerships](https://about.gitlab.com/partners/integrate/index.md).
+1. [Create an issue](https://gitlab.com/gitlab-com/alliances/alliances/issues/new?issuable_template=new_partner)
+   using our new partner issue template to begin the discussion.
+1. Get a test account to begin developing your integration. You can
+   request a [GitLab.com Gold Subscription Sandbox](https://about.gitlab.com/partners/integrate/index.md#gitlabcom-gold-subscription-sandbox-request)
+   or an [EE Developer License](https://about.gitlab.com/partners/integrate/index.md#requesting-ee-dev-license-for-rd).
+1. Provide a [pipeline job](../../development/pipelines.md)
+   template that users could integrate into their own GitLab pipelines.
+1. Create a report artifact with your pipeline jobs.
+1. Ensure your pipeline jobs create a report artifact that GitLab can process
+   to successfully display your own product's results with the rest of GitLab.
+   - See detailed [technical directions](secure.md) for this step.
+   - Read more about [job report artifacts](../../ci/yaml/README.md#artifactsreports).
+   - Read about [job artifacts](../../user/project/pipelines/job_artifacts.md).
+   - Your report artifact must be in one of our currently supported formats.
+     For more information, see the [documentation on reports](secure.md#report).
+      - Documentation for [SAST reports](../../user/application_security/sast/index.md#reports-json-format).
+      - Documentation for [Dependency Scanning reports](../../user/application_security/dependency_scanning/index.md#reports-json-format).
+      - Documentation for [Container Scanning reports](../../user/application_security/container_scanning/index.md#reports-json-format).
+      - See this [example secure job definition that also defines the artifact created](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml).
+      - If you need a new kind of scan or report, [create an issue](https://gitlab.com/gitlab-org/gitlab/issues/new#)
+         and add the label `devops::secure`.
+   - Once the job is completed, the data can be seen:
+      - In the [Merge Request Security Report](../../user/project/merge_requests/index.md#security-reports-ultimate) ([MR Security Report data flow](https://gitlab.com/snippets/1910005#merge-request-view)).
+      - While [browsing a Job Artifact](../../user/project/pipelines/job_artifacts.md).
+      - In the [Security Dashboard](../../user/application_security/security_dashboard/index.md) ([Dashboard data flow](https://gitlab.com/snippets/1910005#project-and-group-dashboards)).
+1. Optional: Provide a way to interact with results as Vulnerabilities:
+   - Users can interact with the findings from your artifact within their workflow. They can dismiss the findings or accept them and create a backlog issue.
+   - To automatically create issues without user interaction, use the [issue API](../../api/issues.md). This will be replaced by [Standalone Vulnerabilities](https://gitlab.com/groups/gitlab-org/-/epics/634) in the future.
+1. Optional: Provide auto-remediation steps:
+   - If you specified `remediations` in your artifact, it is proposed through our [auto-remediation](../../user/application_security/index.md#solutions-for-vulnerabilities-auto-remediation)
+   interface.
+1. Demo the integration to GitLab:
+   - After you have tested and are ready to demo your integration please
+   [reach out](https://about.gitlab.com/partners/integrate/index.md) to us. If you
+   skip this step you won’t be able to do supported marketing.
+1. Begin doing supported marketing of your GitLab integration.
+   - Work with our [partner team](https://about.gitlab.com/partners/integrate/index.md)
+   to support your go-to-market as appropriate.
+   - Examples of supported marketing could include being listed on our [Security Partner page](https://about.gitlab.com/partners/index.md#security),
+   doing an [Unfiltered blog post](https://about.gitlab.com/handbook/marketing/blog/unfiltered/index.md),
+   doing a co-branded webinar, or producing a co-branded whitepaper.
+
+If you have any issues while working through your integration or the steps
+above, please create an issue to discuss with us further.

doc/install/installation.md

@@ -457,16 +457,13 @@ sudo chmod -R u+rwX shared/artifacts/
 # Change the permissions of the directory where GitLab Pages are stored
 sudo chmod -R ug+rwX shared/pages/
-# Copy the example Unicorn config
-sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
-# Find number of cores
-nproc
-
-# Enable cluster mode if you expect to have a high load instance
-# Set the number of workers to at least the number of cores
-# Ex. change the amount of workers to 3 for 2GB RAM server
-sudo -u git -H editor config/unicorn.rb
+# Copy the example Puma config
+sudo -u git -H cp config/puma.rb.example config/puma.rb
+# Refer to https://github.com/puma/puma#configuration for more information.
+# You should scale Puma workers and threads based on the number of CPU
+# cores you have available. You can get that number via the `nproc` command.
+sudo -u git -H editor config/puma.rb
 # Copy the example Rack attack config
 sudo -u git -H cp config/initializers/rack_attack.rb.example config/initializers/rack_attack.rb
@@ -495,8 +492,8 @@ sudo -u git -H editor config/resque.yml
 ```
 CAUTION: **Caution:**
-Make sure to edit both `gitlab.yml` and `unicorn.rb` to match your setup.
-If you want to use Puma web server, see [Using Puma](#using-puma) for the additional steps.
+Make sure to edit both `gitlab.yml` and `puma.rb` to match your setup.
+If you want to use the Unicorn web server, see [Using Unicorn](#using-unicorn) for the additional steps.
 NOTE: **Note:**
 If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional steps.
@@ -947,23 +944,22 @@ You also need to change the corresponding options (e.g. `ssh_user`, `ssh_host`,
 Apart from the always supported Markdown style, there are other rich text files that GitLab can display. But you might have to install a dependency to do so. See the [`github-markup` gem README](https://github.com/gitlabhq/markup#markups) for more information.
-### Using Puma
-
-Puma is a multi-threaded HTTP 1.1 server for Ruby applications.
-To use GitLab with Puma:
-1. Finish GitLab setup so you have it up and running.
-1. Copy the supplied example Puma config file into place:
+### Using Unicorn
+As of GitLab 12.9, [Puma](https://github.com/puma/puma) has replaced Unicorn as the default web server for installations from source.
+If you want to switch back to Unicorn, follow these steps:
+1. Finish the GitLab setup so you have it up and running.
+1. Copy the supplied example Unicorn config file into place:
    ```shell
    cd /home/git/gitlab
    # Copy config file for the web server
-   sudo -u git -H cp config/puma.rb.example config/puma.rb
+   sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
    ```
-1. Edit the system `init.d` script to use `EXPERIMENTAL_PUMA=1` flag. If you have `/etc/default/gitlab`, then you should edit it instead.
+1. Edit the system `init.d` script to set the `USE_UNICORN=1` flag. If you have `/etc/default/gitlab`, then you should edit it instead.
 1. Restart GitLab.
 ## Troubleshooting

doc/user/group/roadmap/index.md

@@ -14,7 +14,7 @@ shows such a visualization for all the epics which are under a group and/or its
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5164) in GitLab 12.9.
 On the epic bars, you can see their title, progress, and completed weight percentage.
-When you hover over an epic bar, a popover appears with its description, start and due dates, and weight completed.
+When you hover over an epic bar, a popover appears with its title, start and due dates, and weight completed.
 ![roadmap view](img/roadmap_view_v12_9.png)

lib/gitlab/checks/post_push_message.rb

@@ -3,8 +3,8 @@
 module Gitlab
   module Checks
     class PostPushMessage
-      def initialize(project, user, protocol)
-        @project = project
+      def initialize(repository, user, protocol)
+        @repository = repository
         @user = user
         @protocol = protocol
       end
@@ -34,14 +34,21 @@ module Gitlab
       protected
-      attr_reader :project, :user, :protocol
+      attr_reader :repository, :user, :protocol
+
+      delegate :project, to: :repository, allow_nil: true
+      delegate :container, to: :repository, allow_nil: false
       def self.message_key(user_id, project_id)
         raise NotImplementedError
       end
       def url_to_repo
-        protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo
+        protocol == 'ssh' ? message_subject.ssh_url_to_repo : message_subject.http_url_to_repo
+      end
+
+      def message_subject
+        repository.repo_type.wiki? ? project.wiki : container
       end
     end
   end

lib/gitlab/checks/project_moved.rb

@@ -5,10 +5,10 @@ module Gitlab
     class ProjectMoved < PostPushMessage
       REDIRECT_NAMESPACE = "redirect_namespace"
-      def initialize(project, user, protocol, redirected_path)
+      def initialize(repository, user, protocol, redirected_path)
         @redirected_path = redirected_path
-        super(project, user, protocol)
+        super(repository, user, protocol)
       end
       def message

lib/gitlab/git_access.rb

@@ -188,7 +188,7 @@ module Gitlab
     def add_project_moved_message!
       return if redirected_path.nil?
-      project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path)
+      project_moved = Checks::ProjectMoved.new(repository, user, protocol, redirected_path)
       project_moved.add_message
     end
@@ -250,7 +250,7 @@ module Gitlab
       @project = project
       user_access.project = @project
-      Checks::ProjectCreated.new(project, user, protocol).add_message
+      Checks::ProjectCreated.new(repository, user, protocol).add_message
     end
     def check_repository_existence!

lib/gitlab/git_access_snippet.rb

@@ -39,11 +39,11 @@ module Gitlab
     override :check_project!
     def check_project!(cmd, changes)
-      if snippet.is_a?(ProjectSnippet)
-        check_namespace!
-        check_project_accessibility!
-        # TODO add add_project_moved_message! to handle non-project repo https://gitlab.com/gitlab-org/gitlab/issues/205646
-      end
+      return unless snippet.is_a?(ProjectSnippet)
+
+      check_namespace!
+      check_project_accessibility!
+      add_project_moved_message!
     end
     override :check_push_access!

lib/gitlab/repo_path.rb

@@ -19,8 +19,7 @@ module Gitlab
         # Removing the suffix (.wiki, .design, ...) from the project path
         full_path = repo_path.chomp(type.path_suffix)
-        container, project, was_redirected = find_container(type, full_path)
-        redirected_path = repo_path if was_redirected
+        container, project, redirected_path = find_container(type, full_path)
         return [container, project, type, redirected_path] if container
       end
@@ -33,22 +32,23 @@ module Gitlab
     def self.find_container(type, full_path)
       if type.snippet?
-        snippet, was_redirected = find_snippet(full_path)
-        [snippet, snippet&.project, was_redirected]
+        snippet, redirected_path = find_snippet(full_path)
+        [snippet, snippet&.project, redirected_path]
       else
-        project, was_redirected = find_project(full_path)
-        [project, project, was_redirected]
+        project, redirected_path = find_project(full_path)
+        [project, project, redirected_path]
       end
     end
     def self.find_project(project_path)
-      return [nil, false] if project_path.blank?
+      return [nil, nil] if project_path.blank?
       project = Project.find_by_full_path(project_path, follow_redirects: true)
-      [project, redirected?(project, project_path)]
+      redirected_path = redirected?(project, project_path) ? project_path : nil
+      [project, redirected_path]
     end
     def self.redirected?(project, project_path)
@@ -59,12 +59,12 @@ module Gitlab
     # - snippets/1
     # - h5bp/html5-boilerplate/snippets/53
     def self.find_snippet(snippet_path)
-      return [nil, false] if snippet_path.blank?
+      return [nil, nil] if snippet_path.blank?
       snippet_id, project_path = extract_snippet_info(snippet_path)
-      project, was_redirected = find_project(project_path)
-      [Snippet.find_by_id_and_project(id: snippet_id, project: project), was_redirected]
+      project, redirected_path = find_project(project_path)
+      [Snippet.find_by_id_and_project(id: snippet_id, project: project), redirected_path]
     end
     def self.extract_snippet_info(snippet_path)

lib/support/init.d/gitlab

@@ -26,7 +26,7 @@
 ### Environment variables
 RAILS_ENV="production"
-EXPERIMENTAL_PUMA=""
+USE_UNICORN=""
 # Script variable names should be lower-case not to conflict with
 # internal /bin/sh variables such as PATH, EDITOR or SHELL.
@@ -68,10 +68,10 @@ if ! cd "$app_root" ; then
 fi
 # Select the web server to use
-if [ -z "$EXPERIMENTAL_PUMA" ]; then
-  use_web_server="unicorn"
-else
+if [ -z "$USE_UNICORN" ]; then
   use_web_server="puma"
+else
+  use_web_server="unicorn"
 fi