Add latest changes from gitlab-org/gitlab@master

doc/user/project/merge_requests/accessibility_testing.md

@@ -21,32 +21,34 @@ analyzed to a file called `accessibility`.
 ## Configure Accessibility Testing
 This example shows how to run [pa11y](https://pa11y.org/)
-on your code with GitLab CI/CD using a node Docker image.
-For GitLab 12.8 and later, to define the `a11y` job, you must
+on your code with GitLab CI/CD using the [GitLab Accessibility Docker image](https://gitlab.com/gitlab-org/ci-cd/accessibility).
+For GitLab 12.9 and later, to define the `a11y` job, you must
 [include](../../../ci/yaml/README.md#includetemplate) the
 [`Accessibility.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml)
 included with your GitLab installation, as shown below.
-For GitLab versions earlier than 12.8, you can copy and use the job as
-defined in that template.
 Add the following to your `.gitlab-ci.yml` file:
 ```yaml
 variables:
-  a11y_urls: "https://about.gitlab.com"
+  a11y_urls: "https://about.gitlab.com https://gitlab.com/users/sign_in"
 include:
-  - remote: "https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml"
+  - template: "Verify/Accessibility.gitlab-ci.yml"
 ```
-The example above will create an `a11y` job in your CI/CD pipeline and will run
-Pa11y against the webpage you defined in `a11y_urls` to build a report.
-NOTE: **Note:**
-Only one URL may be currently passed into `a11y_urls`.
-The full HTML Pa11y report will be saved as an artifact that can be [viewed directly in your browser](../../../ci/pipelines/job_artifacts.md#browsing-artifacts).
+creates an `a11y` job in your CI/CD pipeline, runs
+Pa11y against the webpages defined in `a11y_urls`, and builds an HTML report for each.
+The report for each URL is saved as an artifact that can be [viewed directly in your browser](../../../ci/pipelines/job_artifacts.md#browsing-artifacts).
+
+A single `accessibility.json` artifact is created and saved along with the individual HTML reports.
+It includes report data for all URLs scanned.
+NOTE: **Note:**
+For GitLab versions earlier than 12.9, you can use `include:remote` and use a
+link to the [current template in `master`](https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml)
 NOTE: **Note:**
 The job definition provided by the template does not support Kubernetes yet.

doc/user/reserved_names.md

@@ -69,7 +69,6 @@ Currently the following names are reserved as top level groups:
 - `invites`
 - `jwt`
 - `login`
-- `notification_settings`
 - `oauth`
 - `profile`
 - `projects`

lib/gitlab/background_migration/user_mentions/models/commit.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        class Commit
+          include Concerns::IsolatedMentionable
+          include Concerns::MentionableMigrationMethods
+
+          def self.user_mention_model
+            Gitlab::BackgroundMigration::UserMentions::Models::CommitUserMention
+          end
+
+          def user_mention_model
+            self.class.user_mention_model
+          end
+
+          def user_mention_resource_id
+            id
+          end
+
+          def user_mention_note_id
+            'NULL'
+          end
+
+          def self.no_quote_columns
+            [:note_id]
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        class CommitUserMention < ActiveRecord::Base
+          self.table_name = 'commit_user_mentions'
+
+          def self.resource_foreign_key
+            :commit_id
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/background_migration/user_mentions/models/note.rb

@@ -20,7 +20,7 @@ module Gitlab
           belongs_to :project
           def for_personal_snippet?
-            noteable.class.name == 'PersonalSnippet'
+            noteable && noteable.class.name == 'PersonalSnippet'
           end
           def for_project_noteable?
@@ -32,7 +32,7 @@ module Gitlab
           end
           def for_epic?
-            noteable.class.name == 'Epic'
+            noteable && noteable_type == 'Epic'
           end
           def user_mention_resource_id
@@ -43,6 +43,14 @@ module Gitlab
             id
           end
+          def noteable
+            super unless for_commit?
+          end
+
+          def for_commit?
+            noteable_type == "Commit"
+          end
+
           private
           def mentionable_params
@@ -52,6 +60,8 @@ module Gitlab
           end
           def banzai_context_params
+            return {} unless noteable
+
             { group: noteable.group, label_url_method: :group_epics_url }
           end
         end

lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml

@@ -8,20 +8,12 @@ stages:
 a11y:
   stage: accessibility
-  image: node
-  script:
-    - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
-    - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list && \
-    - apt-get update && \
-    - apt-get install -y google-chrome-stable && \
-    - rm -rf /var/lib/apt/lists/*
-    - npm install pa11y@5.3.0 pa11y-reporter-html@1.0.0
-    - 'echo { \"chromeLaunchConfig\": { \"args\": [\"--no-sandbox\"] }, \"includeWarnings\": true, \"reporter\": \"html\" } > pa11y.json'
-    - './node_modules/.bin/pa11y $a11y_urls > accessibility.html'
+  image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.2
+  script: /gitlab-accessibility.sh $a11y_urls
   allow_failure: true
   artifacts:
     when: always
-    expose_as: 'accessibility'
-    paths: ['accessibility.html']
+    expose_as: 'Accessibility Reports'
+    paths: ['reports/']
   rules:
     - if: $a11y_urls

lib/gitlab/elasticsearch/logs.rb

@@ -3,6 +3,8 @@
 module Gitlab
   module Elasticsearch
     class Logs
+      InvalidCursor = Class.new(RuntimeError)
+
       # How many log lines to fetch in a query
       LOGS_LIMIT = 500
@@ -10,7 +12,7 @@ module Gitlab
         @client = client
       end
-      def pod_logs(namespace, pod_name, container_name = nil, search = nil, start_time = nil, end_time = nil)
+      def pod_logs(namespace, pod_name, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil)
         query = { bool: { must: [] } }.tap do |q|
           filter_pod_name(q, pod_name)
           filter_namespace(q, namespace)
@@ -19,7 +21,7 @@ module Gitlab
           filter_times(q, start_time, end_time)
         end
-        body = build_body(query)
+        body = build_body(query, cursor)
         response = @client.search body: body
         format_response(response)
@@ -27,8 +29,8 @@ module Gitlab
       private
-      def build_body(query)
-        {
+      def build_body(query, cursor = nil)
+        body = {
           query: query,
           # reverse order so we can query N-most recent records
           sort: [
@@ -40,6 +42,12 @@ module Gitlab
           # fixed limit for now, we should support paginated queries
           size: ::Gitlab::Elasticsearch::Logs::LOGS_LIMIT
         }
+
+        unless cursor.nil?
+          body[:search_after] = decode_cursor(cursor)
+        end
+
+        body
       end
       def filter_pod_name(query, pod_name)
@@ -100,7 +108,9 @@ module Gitlab
       end
       def format_response(response)
-        result = response.fetch("hits", {}).fetch("hits", []).map do |hit|
+        results = response.fetch("hits", {}).fetch("hits", [])
+        last_result = results.last
+        results = results.map do |hit|
           {
             timestamp: hit["_source"]["@timestamp"],
             message: hit["_source"]["message"]
@@ -108,7 +118,32 @@ module Gitlab
         end
         # we queried for the N-most recent records but we want them ordered oldest to newest
-        result.reverse
+        {
+          logs: results.reverse,
+          cursor: last_result.nil? ? nil : encode_cursor(last_result["sort"])
+        }
+      end
+
+      # we want to hide the implementation details of the search_after parameter from the frontend
+      # behind a single easily transmitted value
+      def encode_cursor(obj)
+        obj.join(',')
+      end
+
+      def decode_cursor(obj)
+        cursor = obj.split(',').map(&:to_i)
+
+        unless valid_cursor(cursor)
+          raise InvalidCursor, "invalid cursor format"
+        end
+
+        cursor
+      end
+
+      def valid_cursor(cursor)
+        cursor.instance_of?(Array) &&
+        cursor.length == 2 &&
+        cursor.map {|i| i.instance_of?(Integer)}.reduce(:&)
       end
     end
   end

lib/gitlab/git/repository.rb

@@ -152,6 +152,12 @@ module Gitlab
         end
       end
+      def replicate(source_repository)
+        wrapped_gitaly_errors do
+          gitaly_repository_client.replicate(source_repository)
+        end
+      end
+
       def expire_has_local_branches_cache
         clear_memoization(:has_local_branches)
       end
@@ -767,12 +773,6 @@ module Gitlab
         !has_visible_content?
       end
-      def fetch_repository_as_mirror(repository)
-        wrapped_gitaly_errors do
-          gitaly_remote_client.fetch_internal_remote(repository)
-        end
-      end
-
       # Fetch remote for repository
       #
       # remote - remote name

lib/gitlab/gitaly_client/remote_service.rb

@@ -41,20 +41,6 @@ module Gitlab
         GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result
       end
-      def fetch_internal_remote(repository)
-        request = Gitaly::FetchInternalRemoteRequest.new(
-          repository: @gitaly_repo,
-          remote_repository: repository.gitaly_repository
-        )
-
-        response = GitalyClient.call(@storage, :remote_service,
-                                     :fetch_internal_remote, request,
-                                     timeout: GitalyClient.long_timeout,
-                                     remote_storage: repository.storage)
-
-        response.result
-      end
-
       def find_remote_root_ref(remote_name)
         request = Gitaly::FindRemoteRootRefRequest.new(
           repository: @gitaly_repo,

lib/gitlab/gitaly_client/repository_service.rb

@@ -359,6 +359,22 @@ module Gitlab
         GitalyClient.call(@storage, :repository_service, :remove_repository, request, timeout: GitalyClient.long_timeout)
       end
+      def replicate(source_repository)
+        request = Gitaly::ReplicateRepositoryRequest.new(
+          repository: @gitaly_repo,
+          source: source_repository.gitaly_repository
+        )
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          :replicate_repository,
+          request,
+          remote_storage: source_repository.storage,
+          timeout: GitalyClient.long_timeout
+        )
+      end
+
       private
       def search_results_from_response(gitaly_response, options = {})

lib/gitlab/jira/http_client.rb

@@ -15,7 +15,7 @@ module Gitlab
         unless result.response.is_a?(Net::HTTPSuccess)
           Gitlab::ErrorTracking.track_and_raise_exception(
             JIRA::HTTPError.new(result.response),
-            response_body: result.body
+            response: result.body
           )
         end

lib/gitlab/path_regex.rb

@@ -42,7 +42,6 @@ module Gitlab
       invites
       jwt
       login
-      notification_settings
       oauth
       profile
       projects

lib/gitlab/setup_helper.rb

@@ -33,8 +33,6 @@ module Gitlab
         if Rails.env.test?
           storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s
-
-          FileUtils.mkdir(storage_path) unless File.exist?(storage_path)
           storages << { name: 'test_second_storage', path: storage_path }
         end

lib/gitlab/x509/commit.rb

@@ -184,11 +184,13 @@ module Gitlab
           commit_sha: @commit.sha,
           project: @commit.project,
           x509_certificate_id: certificate.id,
-          verification_status: verification_status
+          verification_status: verification_status(certificate)
         }
       end
-      def verification_status
+      def verification_status(certificate)
+        return :unverified if certificate.revoked?
+
         if verified_signature && certificate_email == @commit.committer_email
           :verified
         else

locale/gitlab.pot

@@ -560,6 +560,9 @@ msgstr ""
 msgid "(removed)"
 msgstr ""
+msgid "(revoked)"
+msgstr ""
+
 msgid "*"
 msgstr ""
@@ -10889,6 +10892,9 @@ msgstr ""
 msgid "Invalid URL"
 msgstr ""
+msgid "Invalid cursor value provided"
+msgstr ""
+
 msgid "Invalid date"
 msgstr ""
@@ -21370,9 +21376,6 @@ msgstr ""
 msgid "UpdateRepositoryStorage|Error moving repository storage for %{project_full_path} - %{message}"
 msgstr ""
-msgid "UpdateRepositoryStorage|Failed to fetch %{type} repository as mirror"
-msgstr ""
-
 msgid "UpdateRepositoryStorage|Failed to verify %{type} repository checksum from %{old} to %{new}"
 msgstr ""

spec/factories/x509_certificate.rb

@@ -8,5 +8,6 @@ FactoryBot.define do
     email { 'gitlab@example.org' }
     serial_number { 278969561018901340486471282831158785578 }
     x509_issuer
+    certificate_status { :good }
   end
 end

spec/fixtures/lib/elasticsearch/query_with_cursor.json

+{
+  "query": {
+    "bool": {
+      "must": [
+        {
+          "match_phrase": {
+            "kubernetes.pod.name": {
+              "query": "production-6866bc8974-m4sk4"
+            }
+          }
+        },
+        {
+          "match_phrase": {
+            "kubernetes.namespace": {
+              "query": "autodevops-deploy-9-production"
+            }
+          }
+        }
+      ]
+    }
+  },
+  "sort": [
+    {
+      "@timestamp": {
+        "order": "desc"
+      }
+    },
+    {
+      "offset": {
+        "order": "desc"
+      }
+    }
+  ],
+  "search_after": [
+    9999934,
+    1572449784442
+  ],
+  "_source": [
+    "@timestamp",
+    "message"
+  ],
+  "size": 500
+}

spec/frontend/monitoring/embed/mock_data.js

-export const metricsWithData = [15, 16];
+export const metricsWithData = ['15_metric_a', '16_metric_b'];
 export const groups = [
   {
@@ -7,41 +7,12 @@ export const groups = [
         title: 'Memory Usage (Total)',
         type: 'area-chart',
         y_label: 'Total Memory Used',
-        weight: 4,
-        metrics: [
-          {
-            id: 'system_metrics_kubernetes_container_memory_total',
-            metric_id: 15,
-          },
-        ],
-      },
-      {
-        title: 'Core Usage (Total)',
-        type: 'area-chart',
-        y_label: 'Total Cores',
-        weight: 3,
-        metrics: [
-          {
-            id: 'system_metrics_kubernetes_container_cores_total',
-            metric_id: 16,
-          },
-        ],
+        metrics: null,
       },
     ],
   },
 ];
-export const metrics = [
-  {
-    id: 'system_metrics_kubernetes_container_memory_total',
-    metric_id: 15,
-  },
-  {
-    id: 'system_metrics_kubernetes_container_cores_total',
-    metric_id: 16,
-  },
-];
-
 const result = [
   {
     values: [
@@ -60,7 +31,7 @@ export const metricsData = [
   {
     metrics: [
       {
-        metric_id: 15,
+        metricId: '15_metric_a',
         result,
       },
     ],
@@ -68,7 +39,7 @@ export const metricsData = [
   {
     metrics: [
       {
-        metric_id: 16,
+        metricId: '16_metric_b',
         result,
       },
     ],

spec/frontend/monitoring/store/utils_spec.js

@@ -213,20 +213,16 @@ describe('mapToDashboardViewModel', () => {
       expect(getMappedMetric(dashboard)).toEqual({
         label: expect.any(String),
         metricId: expect.any(String),
-        metric_id: expect.any(String),
       });
     });
-    it('creates a metric with a correct ids', () => {
+    it('creates a metric with a correct id', () => {
       const dashboard = dashboardWithMetric({
         id: 'http_responses',
         metric_id: 1,
       });
-      expect(getMappedMetric(dashboard)).toMatchObject({
-        metricId: '1_http_responses',
-        metric_id: '1_http_responses',
-      });
+      expect(getMappedMetric(dashboard).metricId).toEqual('1_http_responses');
     });
     it('creates a metric with a default label', () => {

spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb

@@ -149,10 +149,12 @@ describe Gitlab::BackgroundMigration::LegacyUploadMover, schema: :latest do
     context 'when an upload belongs to a legacy_diff_note' do
       let!(:merge_request) { create(:merge_request, source_project: project) }
+
       let!(:note) do
         create(:legacy_diff_note_on_merge_request,
           note: 'some note', project: project, noteable: merge_request)
       end
+
       let(:legacy_upload) do
         create(:upload, :with_file, :attachment_upload,
                path: "uploads/-/system/note/attachment/#{note.id}/#{filename}", model: note)
@@ -193,6 +195,17 @@ describe Gitlab::BackgroundMigration::LegacyUploadMover, schema: :latest do
         it_behaves_like 'move error'
       end
+      context 'when upload has mount_point nil' do
+        let(:legacy_upload) do
+          create(:upload, :with_file, :attachment_upload,
+                 path: "uploads/-/system/note/attachment/#{note.id}/#{filename}", model: note, mount_point: nil)
+        end
+
+        it_behaves_like 'migrates the file correctly'
+        it_behaves_like 'legacy local file'
+        it_behaves_like 'legacy upload deletion'
+      end
+
       context 'when the file can be handled correctly' do
         it_behaves_like 'migrates the file correctly'
         it_behaves_like 'legacy local file'