Add latest changes from gitlab-org/gitlab@master

app/assets/javascripts/api.js

@@ -476,12 +476,12 @@ const Api = {
     return axios.get(url);
   },
-  lsifData(projectPath, commitId, path) {
+  lsifData(projectPath, commitId, paths) {
     const url = Api.buildUrl(this.lsifPath)
       .replace(':id', encodeURIComponent(projectPath))
       .replace(':commit_id', commitId);
-    return axios.get(url, { params: { path } });
+    return axios.get(url, { params: { paths } });
   },
   environments(id) {

app/assets/javascripts/code_navigation/store/actions.js

@@ -13,9 +13,10 @@ export default {
     commit(types.REQUEST_DATA);
     api
-      .lsifData(state.projectPath, state.commitId, state.blobPath)
+      .lsifData(state.projectPath, state.commitId, [state.blobPath])
       .then(({ data }) => {
-        const normalizedData = data.reduce((acc, d) => {
+        const dataForPath = data[state.blobPath];
+        const normalizedData = dataForPath.reduce((acc, d) => {
           if (d.hover) {
             acc[`${d.start_line}:${d.start_char}`] = d;
             addInteractionClass(d);

app/controllers/concerns/cycle_analytics_params.rb

@@ -38,7 +38,7 @@ module CycleAnalyticsParams
   end
   def to_utc_time(field)
-    date = field.is_a?(Date) ? field : Date.parse(field)
+    date = field.is_a?(Date) || field.is_a?(Time) ? field : Date.parse(field)
     date.to_time.utc
   end
 end

app/graphql/resolvers/projects/snippets_resolver.rb

@@ -10,6 +10,11 @@ module Resolvers
       def resolve(**args)
         return Snippet.none if project.nil?
+        unless project.feature_available?(:snippets, current_user)
+          raise Gitlab::Graphql::Errors::ResourceNotAvailable,
+            'Snippets are not enabled for this Project'
+        end
+
         super
       end

app/models/project.rb

@@ -2039,6 +2039,16 @@ class Project < ApplicationRecord
     end
   end
+  def change_repository_storage(new_repository_storage_key)
+    return if repository_read_only?
+    return if repository_storage == new_repository_storage_key
+
+    raise ArgumentError unless ::Gitlab.config.repositories.storages.key?(new_repository_storage_key)
+
+    run_after_commit { ProjectUpdateRepositoryStorageWorker.perform_async(id, new_repository_storage_key) }
+    self.repository_read_only = true
+  end
+
   def pushes_since_gc
     Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i }
   end

app/policies/project_policy.rb

@@ -469,6 +469,8 @@ class ProjectPolicy < BasePolicy
     prevent :create_pipeline
   end
+  rule { admin }.enable :change_repository_storage
+
   private
   def team_member?

app/presenters/projects/prometheus/alert_presenter.rb

@@ -53,7 +53,7 @@ module Projects
           #### Summary
           #{metadata_list}
-          #{alert_details}
+          #{alert_details}#{metric_embed_for_alert}
         MARKDOWN
       end
@@ -118,6 +118,10 @@ module Projects
       def host_links
         Array(hosts.value).join(' ')
       end
+
+      def metric_embed_for_alert; end
     end
   end
 end
+
+Projects::Prometheus::AlertPresenter.prepend_if_ee('EE::Projects::Prometheus::AlertPresenter')

app/presenters/snippet_blob_presenter.rb

@@ -3,18 +3,15 @@
 class SnippetBlobPresenter < BlobPresenter
   def rich_data
     return if blob.binary?
-    if markup?
-      blob.rendered_markup
-    else
-      highlight(plain: false)
-    end
+    return unless blob.rich_viewer
+    render_rich_partial
   end
   def plain_data
     return if blob.binary?
-    highlight(plain: !markup?)
+    highlight(plain: false)
   end
   def raw_path
@@ -27,10 +24,6 @@ class SnippetBlobPresenter < BlobPresenter
   private
-  def markup?
-    blob.rich_viewer&.partial_name == 'markup'
-  end
-
   def snippet
     blob.container
   end
@@ -38,4 +31,18 @@ class SnippetBlobPresenter < BlobPresenter
   def language
     nil
   end
+
+  def render_rich_partial
+    renderer.render("projects/blob/viewers/_#{blob.rich_viewer.partial_name}",
+                    locals: { viewer: blob.rich_viewer, blob: blob, blob_raw_path: raw_path },
+                    layout: false)
+  end
+
+  def renderer
+    proxy = Warden::Proxy.new({}, Warden::Manager.new({})).tap do |proxy_instance|
+      proxy_instance.set_user(current_user, scope: :user)
+    end
+
+    ApplicationController.renderer.new('warden' => proxy)
+  end
 end

app/services/projects/container_repository/cleanup_tags_service.rb

@@ -61,10 +61,15 @@ module Projects
       end
       def filter_by_name(tags)
-        regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
+        # Technical Debt: https://gitlab.com/gitlab-org/gitlab/issues/207267
+        # name_regex to be removed when container_expiration_policies is updated
+        # to have both regex columns
+        regex_delete = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_delete'] || params['name_regex']}\\z")
+        regex_retain = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_keep']}\\z")
         tags.select do |tag|
-          regex.scan(tag.name).any?
+          # regex_retain will override any overlapping matches by regex_delete
+          regex_delete.match?(tag.name) && !regex_retain.match?(tag.name)
         end
       end

app/services/projects/lsif_data_service.rb

@@ -2,20 +2,22 @@
 module Projects
   class LsifDataService
-    attr_reader :file, :project, :path, :commit_id,
-      :docs, :doc_ranges, :ranges, :def_refs, :hover_refs
+    attr_reader :file, :project, :commit_id, :docs,
+      :doc_ranges, :ranges, :def_refs, :hover_refs
     CACHE_EXPIRE_IN = 1.hour
-    def initialize(file, project, params)
+    def initialize(file, project, commit_id)
       @file = file
       @project = project
-      @path = params[:path]
-      @commit_id = params[:commit_id]
-    end
-    def execute
+      @commit_id = commit_id
       fetch_data!
+    end
+
+    def execute(path)
+      doc_id = find_doc_id(docs, path)
+      dir_absolute_path = docs[doc_id]&.delete_suffix(path)
       doc_ranges[doc_id]&.map do |range_id|
         location, ref_id = ranges[range_id].values_at('loc', 'ref_id')
@@ -26,7 +28,7 @@ module Projects
           end_line: line_data.last,
           start_char: column_data.first,
           end_char: column_data.last,
-          definition_url: definition_url_for(def_refs[ref_id]),
+          definition_url: definition_url_for(def_refs[ref_id], dir_absolute_path),
           hover: highlighted_hover(hover_refs[ref_id])
         }
       end
@@ -58,8 +60,8 @@ module Projects
       @hover_refs = data['hover_refs']
     end
-    def doc_id
-      @doc_id ||= docs.reduce(nil) do |doc_id, (id, doc_path)|
+    def find_doc_id(docs, path)
+      docs.reduce(nil) do |doc_id, (id, doc_path)|
         next doc_id unless doc_path =~ /#{path}$/
         if doc_id.nil? || docs[doc_id].size > doc_path.size
@@ -70,11 +72,7 @@ module Projects
       end
     end
-    def dir_absolute_path
-      @dir_absolute_path ||= docs[doc_id]&.delete_suffix(path)
-    end
-
-    def definition_url_for(ref_id)
+    def definition_url_for(ref_id, dir_absolute_path)
       return unless range = ranges[ref_id]
       def_doc_id, location = range.values_at('doc_id', 'loc')

app/services/projects/update_repository_storage_service.rb

+# frozen_string_literal: true
+
+module Projects
+  class UpdateRepositoryStorageService < BaseService
+    include Gitlab::ShellAdapter
+
+    RepositoryAlreadyMoved = Class.new(StandardError)
+
+    def initialize(project)
+      @project = project
+    end
+
+    def execute(new_repository_storage_key)
+      # Raising an exception is a little heavy handed but this behavior (doing
+      # nothing if the repo is already on the right storage) prevents data
+      # loss, so it is valuable for us to be able to observe it via the
+      # exception.
+      raise RepositoryAlreadyMoved if project.repository_storage == new_repository_storage_key
+
+      if mirror_repositories(new_repository_storage_key)
+        mark_old_paths_for_archive
+
+        project.update(repository_storage: new_repository_storage_key, repository_read_only: false)
+        project.leave_pool_repository
+        project.track_project_repository
+
+        enqueue_housekeeping
+      else
+        project.update(repository_read_only: false)
+      end
+    end
+
+    private
+
+    def mirror_repositories(new_repository_storage_key)
+      result = mirror_repository(new_repository_storage_key)
+
+      if project.wiki.repository_exists?
+        result &&= mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::WIKI)
+      end
+
+      result
+    end
+
+    def mirror_repository(new_storage_key, type: Gitlab::GlRepository::PROJECT)
+      return false unless wait_for_pushes(type)
+
+      repository = type.repository_for(project)
+      full_path = repository.full_path
+      raw_repository = repository.raw
+
+      # Initialize a git repository on the target path
+      gitlab_shell.create_repository(new_storage_key, raw_repository.relative_path, full_path)
+      new_repository = Gitlab::Git::Repository.new(new_storage_key,
+                                                   raw_repository.relative_path,
+                                                   raw_repository.gl_repository,
+                                                   full_path)
+
+      new_repository.fetch_repository_as_mirror(raw_repository)
+    end
+
+    def mark_old_paths_for_archive
+      old_repository_storage = project.repository_storage
+      new_project_path = moved_path(project.disk_path)
+
+      # Notice that the block passed to `run_after_commit` will run with `project`
+      # as its context
+      project.run_after_commit do
+        GitlabShellWorker.perform_async(:mv_repository,
+                                        old_repository_storage,
+                                        disk_path,
+                                        new_project_path)
+
+        if wiki.repository_exists?
+          GitlabShellWorker.perform_async(:mv_repository,
+                                          old_repository_storage,
+                                          wiki.disk_path,
+                                          "#{new_project_path}.wiki")
+        end
+      end
+    end
+
+    def moved_path(path)
+      "#{path}+#{project.id}+moved+#{Time.now.to_i}"
+    end
+
+    # The underlying FetchInternalRemote call uses a `git fetch` to move data
+    # to the new repository, which leaves it in a less-well-packed state,
+    # lacking bitmaps and commit graphs. Housekeeping will boost performance
+    # significantly.
+    def enqueue_housekeeping
+      return unless Gitlab::CurrentSettings.housekeeping_enabled?
+      return unless Feature.enabled?(:repack_after_shard_migration, project)
+
+      Projects::HousekeepingService.new(project, :gc).execute
+    rescue Projects::HousekeepingService::LeaseTaken
+      # No action required
+    end
+
+    def wait_for_pushes(type)
+      reference_counter = project.reference_counter(type: type)
+
+      # Try for 30 seconds, polling every 10
+      3.times do
+        return true if reference_counter.value == 0
+
+        sleep 10
+      end
+
+      false
+    end
+  end
+end
+
+Projects::UpdateRepositoryStorageService.prepend_if_ee('EE::Projects::UpdateRepositoryStorageService')

app/services/projects/update_service.rb

@@ -13,6 +13,10 @@ module Projects
       ensure_wiki_exists if enabling_wiki?
+      if changing_storage_size?
+        project.change_repository_storage(params.delete(:repository_storage))
+      end
+
       yield if block_given?
       validate_classification_label(project, :external_authorization_classification_label)
@@ -140,6 +144,13 @@ module Projects
     def changing_pages_https_only?
       project.previous_changes.include?(:pages_https_only)
     end
+
+    def changing_storage_size?
+      new_repository_storage = params[:repository_storage]
+
+      new_repository_storage && project.repository.exists? &&
+        can?(current_user, :change_repository_storage, project)
+    end
   end
 end

app/workers/all_queues.yml

@@ -1151,6 +1151,13 @@
   :resource_boundary: :unknown
   :weight: 1
   :idempotent: 
+- :name: project_update_repository_storage
+  :feature_category: :source_code_management
+  :has_external_dependencies: 
+  :urgency: :default
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
 - :name: propagate_service_template
   :feature_category: :source_code_management
   :has_external_dependencies: 

app/workers/project_update_repository_storage_worker.rb

+# frozen_string_literal: true
+
+class ProjectUpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker
+  include ApplicationWorker
+
+  feature_category :source_code_management
+
+  def perform(project_id, new_repository_storage_key)
+    project = Project.find(project_id)
+
+    ::Projects::UpdateRepositoryStorageService.new(project).execute(new_repository_storage_key)
+  rescue ::Projects::UpdateRepositoryStorageService::RepositoryAlreadyMoved
+    Rails.logger.info "#{self.class}: repository already moved: #{project}" # rubocop:disable Gitlab/RailsLogger
+  end
+end

changelogs/unreleased/27072-name-regex-allow-bulk-api.yml

+---
+title: Add name_regex_keep param to container registry bulk delete API endpoint
+merge_request: 25484
+author:
+type: added

changelogs/unreleased/37320-ensure-project-snippet-feature-status-in-project-snippet-api-endpoi.yml

+---
+title: Project Snippets GraphQL resolver checks feature status
+merge_request: 26158
+author:
+type: performance

changelogs/unreleased/al-205505-fix-snippet-blob-viewers.yml

+---
+title: Fix snippet blob viewers for rich and plain data
+merge_request: 25945
+author:
+type: fixed

changelogs/unreleased/move-storage-shards.yml

+---
+title: "Backport API support to move between repository storages/shards"
+merge_request: 18721
+author: Ben Bodenmiller
+type: added

changelogs/unreleased/sy-auto-embed-alert.yml

+---
+title: Automatically include embedded metrics for GitLab alert incidents
+merge_request: 25277
+author:
+type: added

doc/api/container_registry.md

@@ -231,7 +231,9 @@ DELETE /projects/:id/registry/repositories/:repository_id/tags
 | --------- | ---- | -------- | ----------- |
 | `id`      | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
 | `repository_id` | integer | yes | The ID of registry repository. |
-| `name_regex` | string | yes | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`.|
+| `name_regex` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`. **Note:** `name_regex` is deprecated in favor of `name_regex_delete`.|
+| `name_regex_delete` | string | yes | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`.|
+| `name_regex_keep` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to keep. This value will override any matches from `name_regex_delete`. Note: setting to `.*` will result in a no-op. |
 | `keep_n` | integer | no | The amount of latest tags of given name to keep. |
 | `older_than` | string | no | Tags to delete that are older than the given time, written in human readable form `1h`, `1d`, `1month`. |
@@ -239,7 +241,7 @@ This API call performs the following operations:
 1. It orders all tags by creation date. The creation date is the time of the
    manifest creation, not the time of tag push.
-1. It removes only the tags matching the given `name_regex`.
+1. It removes only the tags matching the given `name_regex_delete` (or deprecated `name_regex`), keeping any that match `name_regex_keep`.
 1. It never removes the tag named `latest`.
 1. It keeps N latest matching tags (if `keep_n` is specified).
 1. It only removes tags that are older than X amount of time (if `older_than` is specified).
@@ -261,17 +263,23 @@ Examples:
    and remove ones that are older than 2 days:
    ```shell
-   curl --request DELETE --data 'name_regex=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
+   curl --request DELETE --data 'name_regex_delete=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
    ```
 1. Remove all tags, but keep always the latest 5:
    ```shell
-   curl --request DELETE --data 'name_regex=.*' --data 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
+   curl --request DELETE --data 'name_regex_delete=.*' --data 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
+   ```
+
+1. Remove all tags, but keep always tags beginning with `stable`:
+
+   ```shell
+   curl --request DELETE --data 'name_regex_delete=.*' --data 'name_regex_keep=stable.*' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
    ```
 1. Remove all tags that are older than 1 month:
    ```shell
-   curl --request DELETE --data 'name_regex=.*' --data 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
+   curl --request DELETE --data 'name_regex_delete=.*' --data 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
    ```