Add latest changes from gitlab-org/gitlab@master

app/models/environment.rb

@@ -6,6 +6,7 @@ class Environment < ApplicationRecord
   self.reactive_cache_refresh_interval = 1.minute
   self.reactive_cache_lifetime = 55.seconds
+  self.reactive_cache_hard_limit = 10.megabytes
   belongs_to :project, required: true

app/models/merge_request.rb

@@ -24,6 +24,7 @@ class MergeRequest < ApplicationRecord
   self.reactive_cache_key = ->(model) { [model.project.id, model.iid] }
   self.reactive_cache_refresh_interval = 10.minutes
   self.reactive_cache_lifetime = 10.minutes
+  self.reactive_cache_hard_limit = 20.megabytes
   SORTING_PREFERENCE_FIELD = :merge_requests_sort

app/services/projects/lsif_data_service.rb

+# frozen_string_literal: true
+
+module Projects
+  class LsifDataService
+    attr_reader :file, :project, :path, :commit_id
+
+    CACHE_EXPIRE_IN = 1.hour
+
+    def initialize(file, project, params)
+      @file = file
+      @project = project
+      @path = params[:path]
+      @commit_id = params[:commit_id]
+    end
+
+    def execute
+      docs, doc_ranges, ranges =
+        fetch_data.values_at('docs', 'doc_ranges', 'ranges')
+
+      doc_id = doc_id_from(docs)
+
+      doc_ranges[doc_id]&.map do |range_id|
+        line_data, column_data = ranges[range_id]['loc']
+
+        {
+          start_line: line_data.first,
+          end_line: line_data.last,
+          start_char: column_data.first,
+          end_char: column_data.last
+        }
+      end
+    end
+
+    private
+
+    def fetch_data
+      Rails.cache.fetch("project:#{project.id}:lsif:#{commit_id}", expires_in: CACHE_EXPIRE_IN) do
+        data = nil
+
+        file.open do |stream|
+          Zlib::GzipReader.wrap(stream) do |gz_stream|
+            data = JSON.parse(gz_stream.read)
+          end
+        end
+
+        data
+      end
+    end
+
+    def doc_id_from(docs)
+      docs.reduce(nil) do |doc_id, (id, doc_path)|
+        next doc_id unless doc_path =~ /#{path}$/
+
+        if doc_id.nil? || docs[doc_id].size > doc_path.size
+          doc_id = id
+        end
+
+        doc_id
+      end
+    end
+  end
+end

app/services/spam/ham_service.rb

@@ -8,7 +8,7 @@ module Spam
       @spam_log = spam_log
     end
-    def mark_as_ham!
+    def execute
       if akismet.submit_ham
         spam_log.update_attribute(:submitted_as_ham, true)
       else

app/views/groups/registry/repositories/index.html.haml

@@ -3,10 +3,20 @@
 %section
   .row.registry-placeholder.prepend-bottom-10
     .col-12
-      #js-vue-registry-images{ data: { endpoint: group_container_registries_path(@group, format: :json),
-        "help_page_path" => help_page_path('user/packages/container_registry/index'),
-        "no_containers_image" => image_path('illustrations/docker-empty-state.svg'),
-        "containers_error_image" => image_path('illustrations/docker-error-state.svg'),
-        "repository_url" => "",
-        is_group_page: true,
-        character_error: @character_error.to_s } }
+      - if Feature.enabled?(:vue_container_registry_explorer)
+        #js-container-registry{ data: { endpoint: group_container_registries_path(@group),
+          "help_page_path" => help_page_path('user/packages/container_registry/index'),
+          "two_factor_auth_help_link" => help_page_path('user/profile/account/two_factor_authentication'),
+          "personal_access_tokens_help_link" => help_page_path('user/profile/personal_access_tokens'),
+          "no_containers_image" => image_path('illustrations/docker-empty-state.svg'),
+          "containers_error_image" => image_path('illustrations/docker-error-state.svg'),
+          "registry_host_url_with_port" => escape_once(registry_config.host_port),
+          character_error: @character_error.to_s } }
+      - else
+        #js-vue-registry-images{ data: { endpoint: group_container_registries_path(@group, format: :json),
+          "help_page_path" => help_page_path('user/packages/container_registry/index'),
+          "no_containers_image" => image_path('illustrations/docker-empty-state.svg'),
+          "containers_error_image" => image_path('illustrations/docker-error-state.svg'),
+          "repository_url" => "",
+          is_group_page: true,
+          character_error: @character_error.to_s } }

app/views/projects/registry/repositories/index.html.haml

@@ -3,12 +3,24 @@
 %section
   .row.registry-placeholder.prepend-bottom-10
     .col-12
-      #js-vue-registry-images{ data: { endpoint: project_container_registry_index_path(@project, format: :json),
-        "help_page_path" => help_page_path('user/packages/container_registry/index'),
-        "two_factor_auth_help_link" => help_page_path('user/profile/account/two_factor_authentication'),
-        "personal_access_tokens_help_link" => help_page_path('user/profile/personal_access_tokens'),
-        "no_containers_image" => image_path('illustrations/docker-empty-state.svg'),
-        "containers_error_image" => image_path('illustrations/docker-error-state.svg'),
-        "repository_url" => escape_once(@project.container_registry_url),
-        "registry_host_url_with_port" => escape_once(registry_config.host_port),
-        character_error: @character_error.to_s } }
+      - if Feature.enabled?(:vue_container_registry_explorer)
+        #js-container-registry{ data: { endpoint: project_container_registry_index_path(@project),
+          project_path: @project.full_path,
+          "help_page_path" => help_page_path('user/packages/container_registry/index'),
+          "two_factor_auth_help_link" => help_page_path('user/profile/account/two_factor_authentication'),
+          "personal_access_tokens_help_link" => help_page_path('user/profile/personal_access_tokens'),
+          "no_containers_image" => image_path('illustrations/docker-empty-state.svg'),
+          "containers_error_image" => image_path('illustrations/docker-error-state.svg'),
+          "repository_url" => escape_once(@project.container_registry_url),
+          "registry_host_url_with_port" => escape_once(registry_config.host_port),
+          character_error: @character_error.to_s } }
+      - else
+        #js-vue-registry-images{ data: { endpoint: project_container_registry_index_path(@project, format: :json),
+          "help_page_path" => help_page_path('user/packages/container_registry/index'),
+          "two_factor_auth_help_link" => help_page_path('user/profile/account/two_factor_authentication'),
+          "personal_access_tokens_help_link" => help_page_path('user/profile/personal_access_tokens'),
+          "no_containers_image" => image_path('illustrations/docker-empty-state.svg'),
+          "containers_error_image" => image_path('illustrations/docker-error-state.svg'),
+          "repository_url" => escape_once(@project.container_registry_url),
+          "registry_host_url_with_port" => escape_once(registry_config.host_port),
+          character_error: @character_error.to_s } }

app/workers/reactive_caching_worker.rb

@@ -25,5 +25,7 @@ class ReactiveCachingWorker
       .reactive_cache_worker_finder
       .call(id, *args)
       .try(:exclusively_update_reactive_cache!, *args)
+  rescue ReactiveCaching::ExceededReactiveCacheLimit => e
+    Gitlab::ErrorTracking.track_exception(e)
   end
 end

changelogs/unreleased/198652-update-index-on-license-scanning.yml

+---
+title: Include license_scanning to index_ci_builds_on_name_for_security_products_values
+merge_request: 24090
+author:
+type: changed

changelogs/unreleased/feature-set-redis-hard-limits.yml

+---
+title: Sets size limits on data loaded async, like deploy boards and merge request reports
+merge_request: 21871
+author:
+type: changed

changelogs/unreleased/rest_api_log_last_activity_on.yml

+---
+title: Log user last activity on REST API
+merge_request: 21725
+author:
+type: fixed

db/post_migrate/20200206135203_udpate_index_ci_builds_on_name_for_security_products.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class UdpateIndexCiBuildsOnNameForSecurityProducts < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_ci_builds_on_name_for_security_products_values'
+  INDEX_NAME_NEW = 'index_ci_builds_on_name_for_security_reports_values'
+  INITIAL_INDEX = "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text"
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:ci_builds,
+                         :name,
+                         name: INDEX_NAME_NEW,
+                         where: INITIAL_INDEX + ", ('license_scanning'::character varying)::text]))")
+
+    remove_concurrent_index_by_name(:ci_builds, INDEX_NAME)
+  end
+
+  def down
+    add_concurrent_index(:ci_builds,
+                         :name,
+                         name: INDEX_NAME,
+                         where: INITIAL_INDEX + ']))')
+
+    remove_concurrent_index_by_name(:ci_builds, INDEX_NAME_NEW)
+  end
+end

db/schema.rb

@@ -682,7 +682,7 @@ ActiveRecord::Schema.define(version: 2020_02_07_151640) do
     t.index ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type"
     t.index ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref"
     t.index ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref"
-    t.index ["name"], name: "index_ci_builds_on_name_for_security_products_values", where: "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text]))"
+    t.index ["name"], name: "index_ci_builds_on_name_for_security_reports_values", where: "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text, ('license_scanning'::character varying)::text]))"
     t.index ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id"
     t.index ["project_id", "name", "ref"], name: "index_ci_builds_on_project_id_and_name_and_ref", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = 'success'::text) AND ((retried = false) OR (retried IS NULL)))"
     t.index ["project_id", "status"], name: "index_ci_builds_project_id_and_status_for_live_jobs_partial2", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text])))"

doc/administration/instance_limits.md

@@ -87,6 +87,20 @@ Plan.default.limits.update!(ci_active_jobs: 500)
 NOTE: **Note:** Set the limit to `0` to disable it.
+## Environment data on Deploy Boards
+
+[Deploy Boards](../user/project/deploy_boards.md) load information from Kubernetes about
+Pods and Deployments. However, data over 10 MB for a certain environment read from
+Kubernetes won't be shown.
+
+## Merge Request reports
+
+Reports that go over the 20 MB limit won't be loaded. Affected reports:
+
+- [Merge Request security reports](../user/project/merge_requests/index.md#security-reports-ultimate)
+- [CI/CD parameter `artifacts:expose_as`](../ci/yaml/README.md#artifactsexpose_as)
+- [JUnit test reports](../ci/junit_test_reports.md)
+
 ## Advanced Global Search limits
 ### Maximum field length

doc/api/protected_branches.md

@@ -24,6 +24,7 @@ GET /projects/:id/protected_branches
 | Attribute | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
 | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
+| `search` | string | no | Name or part of the name of protected branches to be searched for |
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_branches'

doc/api/users.md

@@ -1376,6 +1376,7 @@ The activities that update the timestamp are:
 - Git HTTP/SSH activities (such as clone, push)
 - User logging in into GitLab
 - User visiting pages related to Dashboards, Projects, Issues and Merge Requests ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/54947) in GitLab 11.8)
+- User using the API
 By default, it shows the activity for all users in the last 6 months, but this can be
 amended by using the `from` parameter.

doc/ci/docker/using_docker_build.md

@@ -217,8 +217,8 @@ support this.
      # The 'docker' hostname is the alias of the service container as described at
      # https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#accessing-the-services.
      #
-     # Note that if you're using the Kubernetes executor, the variable
-     # should be set to tcp://localhost:2376 because of how the
+     # Note that if you're using GitLab Runner 12.7 or earlier with the Kubernetes executor and Kubernetes 1.6 or earlier,
+     # the variable must be set to tcp://localhost:2376 because of how the
      # Kubernetes executor connects services to the job container
      # DOCKER_HOST: tcp://localhost:2376
      #
@@ -279,12 +279,11 @@ variables:
   # The 'docker' hostname is the alias of the service container as described at
   # https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#accessing-the-services
   #
-  # Note that if you're using the Kubernetes executor, the variable should be set to
-  # tcp://localhost:2375 because of how the Kubernetes executor connects services
-  # to the job container
+  # Note that if you're using GitLab Runner 12.7 or earlier with the Kubernetes executor and Kubernetes 1.6 or earlier,
+  # the variable must be set to tcp://localhost:2375 because of how the
+  # Kubernetes executor connects services to the job container
   # DOCKER_HOST: tcp://localhost:2375
   #
-  # For non-Kubernetes executors, we use tcp://docker:2375
   DOCKER_HOST: tcp://docker:2375
   #
   # This will instruct Docker not to start over TLS.

doc/ci/docker/using_docker_images.md

@@ -345,6 +345,9 @@ For example, the following two definitions are equal:
 | `command`    | no       | 9.4 |Command or script that should be used as the container's command. It will be translated to arguments passed to Docker after the image's name. The syntax is similar to [`Dockerfile`'s `CMD`][cmd] directive, where each shell token is a separate string in the array. |
 | `alias`      | no       | 9.4 |Additional alias that can be used to access the service from the job's container. Read [Accessing the services](#accessing-the-services) for more information. |
+NOTE: **Note:**
+Alias support for the Kubernetes executor was [introduced](https://gitlab.com/gitlab-org/gitlab-runner/issues/2229) in GitLab Runner 12.8, and is only available for Kubernetes version 1.7 or later.
+
 ### Starting multiple services from the same image
 > Introduced in GitLab and GitLab Runner 9.4. Read more about the [extended

doc/development/reactive_caching.md

@@ -48,6 +48,12 @@ of the cache by the `reactive_cache_lifetime` value.
 Once the lifetime has expired, no more background jobs will be enqueued and calling
 `#with_reactive_cache` will again return `nil` - starting the process all over again.
+### 1 MB hard limit
+
+`ReactiveCaching` has a 1 megabyte default limit. [This value is configurable](#selfreactive_cache_worker_finder).
+
+If the data we're trying to cache has over 1 megabyte, it will not be cached and a handled `ReactiveCaching::ExceededReactiveCacheLimit` will be notified on Sentry.
+
 ## When to use
 - If we need to make a request to an external API (for example, requests to the k8s API).
@@ -228,6 +234,16 @@ be reset to `reactive_cache_lifetime`.
 self.reactive_cache_lifetime = 10.minutes
 ```
+#### `self.reactive_cache_hard_limit`
+
+- This is the maximum data size that `ReactiveCaching` allows to be cached.
+- The default is 1 megabyte. Data that goes over this value will not be cached
+and will silently raise `ReactiveCaching::ExceededReactiveCacheLimit` on Sentry.
+
+```ruby
+self.reactive_cache_hard_limit = 5.megabytes
+```
+
 #### `self.reactive_cache_worker_finder`
 - This is the method used by the background worker to find or generate the object on

doc/user/instance_statistics/user_cohorts.md

@@ -26,3 +26,4 @@ How do we measure the activity of users? GitLab considers a user active if:
 - The user signs in.
 - The user has Git activity (whether push or pull).
 - The user visits pages related to Dashboards, Projects, Issues and Merge Requests ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/54947) in GitLab 11.8).
+- The user uses the API

lib/api/api.rb

@@ -103,94 +103,102 @@ module API
     helpers ::API::Helpers
     helpers ::API::Helpers::CommonHelpers
-    # Keep in alphabetical order
-    mount ::API::AccessRequests
-    mount ::API::Appearance
-    mount ::API::Applications
-    mount ::API::Avatar
-    mount ::API::AwardEmoji
-    mount ::API::Badges
-    mount ::API::Boards
-    mount ::API::Branches
-    mount ::API::BroadcastMessages
-    mount ::API::Commits
-    mount ::API::CommitStatuses
-    mount ::API::DeployKeys
-    mount ::API::Deployments
-    mount ::API::Environments
-    mount ::API::ErrorTracking
-    mount ::API::Events
-    mount ::API::Features
-    mount ::API::Files
-    mount ::API::GroupBoards
-    mount ::API::GroupClusters
-    mount ::API::GroupExport
-    mount ::API::GroupLabels
-    mount ::API::GroupMilestones
-    mount ::API::Groups
-    mount ::API::GroupContainerRepositories
-    mount ::API::GroupVariables
-    mount ::API::ImportGithub
+    namespace do
+      after do
+        ::Users::ActivityService.new(@current_user).execute if Feature.enabled?(:api_activity_logging)
+      end
+
+      # Keep in alphabetical order
+      mount ::API::AccessRequests
+      mount ::API::Appearance
+      mount ::API::Applications
+      mount ::API::Avatar
+      mount ::API::AwardEmoji
+      mount ::API::Badges
+      mount ::API::Boards
+      mount ::API::Branches
+      mount ::API::BroadcastMessages
+      mount ::API::Commits
+      mount ::API::CommitStatuses
+      mount ::API::DeployKeys
+      mount ::API::Deployments
+      mount ::API::Environments
+      mount ::API::ErrorTracking
+      mount ::API::Events
+      mount ::API::Features
+      mount ::API::Files
+      mount ::API::GroupBoards
+      mount ::API::GroupClusters
+      mount ::API::GroupExport
+      mount ::API::GroupLabels
+      mount ::API::GroupMilestones
+      mount ::API::Groups
+      mount ::API::GroupContainerRepositories
+      mount ::API::GroupVariables
+      mount ::API::ImportGithub
+      mount ::API::Issues
+      mount ::API::JobArtifacts
+      mount ::API::Jobs
+      mount ::API::Keys
+      mount ::API::Labels
+      mount ::API::Lint
+      mount ::API::LsifData
+      mount ::API::Markdown
+      mount ::API::Members
+      mount ::API::MergeRequestDiffs
+      mount ::API::MergeRequests
+      mount ::API::Namespaces
+      mount ::API::Notes
+      mount ::API::Discussions
+      mount ::API::ResourceLabelEvents
+      mount ::API::NotificationSettings
+      mount ::API::Pages
+      mount ::API::PagesDomains
+      mount ::API::Pipelines
+      mount ::API::PipelineSchedules
+      mount ::API::ProjectClusters
+      mount ::API::ProjectContainerRepositories
+      mount ::API::ProjectEvents
+      mount ::API::ProjectExport
+      mount ::API::ProjectImport
+      mount ::API::ProjectHooks
+      mount ::API::ProjectMilestones
+      mount ::API::Projects
+      mount ::API::ProjectSnapshots
+      mount ::API::ProjectSnippets
+      mount ::API::ProjectStatistics
+      mount ::API::ProjectTemplates
+      mount ::API::ProtectedBranches
+      mount ::API::ProtectedTags
+      mount ::API::Releases
+      mount ::API::Release::Links
+      mount ::API::RemoteMirrors
+      mount ::API::Repositories
+      mount ::API::Runner
+      mount ::API::Runners
+      mount ::API::Search
+      mount ::API::Services
+      mount ::API::Settings
+      mount ::API::SidekiqMetrics
+      mount ::API::Snippets
+      mount ::API::Statistics
+      mount ::API::Submodules
+      mount ::API::Subscriptions
+      mount ::API::Suggestions
+      mount ::API::SystemHooks
+      mount ::API::Tags
+      mount ::API::Templates
+      mount ::API::Todos
+      mount ::API::Triggers
+      mount ::API::UserCounts
+      mount ::API::Users
+      mount ::API::Variables
+      mount ::API::Version
+      mount ::API::Wikis
+    end
+
     mount ::API::Internal::Base
     mount ::API::Internal::Pages
-    mount ::API::Issues
-    mount ::API::JobArtifacts
-    mount ::API::Jobs
-    mount ::API::Keys
-    mount ::API::Labels
-    mount ::API::Lint
-    mount ::API::Markdown
-    mount ::API::Members
-    mount ::API::MergeRequestDiffs
-    mount ::API::MergeRequests
-    mount ::API::Namespaces
-    mount ::API::Notes
-    mount ::API::Discussions
-    mount ::API::ResourceLabelEvents
-    mount ::API::NotificationSettings
-    mount ::API::Pages
-    mount ::API::PagesDomains
-    mount ::API::Pipelines
-    mount ::API::PipelineSchedules
-    mount ::API::ProjectClusters
-    mount ::API::ProjectContainerRepositories
-    mount ::API::ProjectEvents
-    mount ::API::ProjectExport
-    mount ::API::ProjectImport
-    mount ::API::ProjectHooks
-    mount ::API::ProjectMilestones
-    mount ::API::Projects
-    mount ::API::ProjectSnapshots
-    mount ::API::ProjectSnippets
-    mount ::API::ProjectStatistics
-    mount ::API::ProjectTemplates
-    mount ::API::ProtectedBranches
-    mount ::API::ProtectedTags
-    mount ::API::Releases
-    mount ::API::Release::Links
-    mount ::API::RemoteMirrors
-    mount ::API::Repositories
-    mount ::API::Runner
-    mount ::API::Runners
-    mount ::API::Search
-    mount ::API::Services
-    mount ::API::Settings
-    mount ::API::SidekiqMetrics
-    mount ::API::Snippets
-    mount ::API::Statistics
-    mount ::API::Submodules
-    mount ::API::Subscriptions
-    mount ::API::Suggestions
-    mount ::API::SystemHooks
-    mount ::API::Tags
-    mount ::API::Templates
-    mount ::API::Todos
-    mount ::API::Triggers
-    mount ::API::UserCounts
-    mount ::API::Users
-    mount ::API::Variables
-    mount ::API::Version
-    mount ::API::Wikis
     route :any, '*path' do
       error!('404 Not Found', 404)