Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/rails.gitlab-ci.yml

@@ -380,13 +380,13 @@ rspec-ee integration pg10:
   extends:
     - .rspec-ee-base-pg10
     - .rails:rules:master-refs-code-backstage
-  parallel: 3
+  parallel: 4
 rspec-ee system pg10:
   extends:
     - .rspec-ee-base-pg10
     - .rails:rules:master-refs-code-backstage
-  parallel: 5
+  parallel: 6
 # ee + master-only jobs #
 #########################

app/graphql/types/snippets/blob_type.rb

@@ -8,11 +8,11 @@ module Types
       description 'Represents the snippet blob'
       present_using SnippetBlobPresenter
-      field :highlighted_data, GraphQL::STRING_TYPE,
+      field :rich_data, GraphQL::STRING_TYPE,
             description: 'Blob highlighted data',
             null: true
-      field :plain_highlighted_data, GraphQL::STRING_TYPE,
+      field :plain_data, GraphQL::STRING_TYPE,
             description: 'Blob plain highlighted data',
             null: true

app/helpers/application_settings_helper.rb

@@ -119,6 +119,17 @@ module ApplicationSettingsHelper
     options_for_select(options, selected)
   end
+  def repository_storages_options_json
+    options = Gitlab.config.repositories.storages.map do |name, storage|
+      {
+        label: "#{name} - #{storage['gitaly_address']}",
+        value: name
+      }
+    end
+
+    options.to_json
+  end
+
   def external_authorization_description
     _("If enabled, access to projects will be validated on an external service"\
         " using their classification label.")

app/models/repository.rb

@@ -65,8 +65,6 @@ class Repository
     xcode_config: :xcode_project?
   }.freeze
-  MERGED_BRANCH_NAMES_CACHE_DURATION = 10.minutes
-
   def initialize(full_path, container, disk_path: nil, repo_type: Gitlab::GlRepository::PROJECT)
     @full_path = full_path
     @disk_path = disk_path || full_path
@@ -914,38 +912,28 @@ class Repository
     @root_ref_sha ||= commit(root_ref).sha
   end
+  # If this method is not provided a set of branch names to check merge status,
+  # it fetches all branches.
   def merged_branch_names(branch_names = [])
     # Currently we should skip caching if requesting all branch names
     # This is only used in a few places, notably app/services/branches/delete_merged_service.rb,
     # and it could potentially result in a very large cache/performance issues with the current
     # implementation.
-    skip_cache = branch_names.empty? || Feature.disabled?(:merged_branch_names_redis_caching)
+    skip_cache = branch_names.empty? || Feature.disabled?(:merged_branch_names_redis_caching, default_enabled: true)
     return raw_repository.merged_branch_names(branch_names) if skip_cache
-    cached_branch_names = cache.read(:merged_branch_names)
-    merged_branch_names_hash = cached_branch_names || {}
-    missing_branch_names = branch_names.select { |bn| !merged_branch_names_hash.key?(bn) }
-
-    # Track some metrics here whilst feature flag is enabled
-    if cached_branch_names.present?
-      counter = Gitlab::Metrics.counter(
-        :gitlab_repository_merged_branch_names_cache_hit,
-        "Count of cache hits for Repository#merged_branch_names"
-      )
-      counter.increment(full_hit: missing_branch_names.empty?)
-    end
-    if missing_branch_names.any?
+    cache = redis_hash_cache
+    merged_branch_names_hash = cache.fetch_and_add_missing(:merged_branch_names, branch_names) do |missing_branch_names, hash|
       merged = raw_repository.merged_branch_names(missing_branch_names)
       missing_branch_names.each do |bn|
-        merged_branch_names_hash[bn] = merged.include?(bn)
+        # Redis only stores strings in hset keys, use a fancy encoder
+        hash[bn] = Gitlab::Redis::Boolean.new(merged.include?(bn))
       end
-
-      cache.write(:merged_branch_names, merged_branch_names_hash, expires_in: MERGED_BRANCH_NAMES_CACHE_DURATION)
     end
-    Set.new(merged_branch_names_hash.select { |_, v| v }.keys)
+    Set.new(merged_branch_names_hash.select { |_, v| Gitlab::Redis::Boolean.true?(v) }.keys)
   end
   def merge_base(*commits_or_ids)
@@ -1168,6 +1156,10 @@ class Repository
     @redis_set_cache ||= Gitlab::RepositorySetCache.new(self)
   end
+  def redis_hash_cache
+    @redis_hash_cache ||= Gitlab::RepositoryHashCache.new(self)
+  end
+
   def request_store_cache
     @request_store_cache ||= Gitlab::RepositoryCache.new(self, backend: Gitlab::SafeRequestStore)
   end

app/presenters/snippet_blob_presenter.rb

 # frozen_string_literal: true
 class SnippetBlobPresenter < BlobPresenter
-  def highlighted_data
+  def rich_data
     return if blob.binary?
-    highlight(plain: false)
+    if markup?
+      blob.rendered_markup
+    else
+      highlight(plain: false)
+    end
   end
-  def plain_highlighted_data
+  def plain_data
     return if blob.binary?
-    highlight(plain: true)
+    highlight(plain: !markup?)
   end
   def raw_path
@@ -23,6 +27,10 @@ class SnippetBlobPresenter < BlobPresenter
   private
+  def markup?
+    blob.rich_viewer&.partial_name == 'markup'
+  end
+
   def snippet
     blob.snippet
   end

app/services/users/destroy_service.rb

@@ -56,12 +56,10 @@ module Users
       MigrateToGhostUserService.new(user).execute unless options[:hard_delete]
-      if Feature.enabled?(:destroy_user_associations_in_batches)
-        # Rails attempts to load all related records into memory before
-        # destroying: https://github.com/rails/rails/issues/22510
-        # This ensures we delete records in batches.
-        user.destroy_dependent_associations_in_batches
-      end
+      # Rails attempts to load all related records into memory before
+      # destroying: https://github.com/rails/rails/issues/22510
+      # This ensures we delete records in batches.
+      user.destroy_dependent_associations_in_batches
       # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing
       user_data = user.destroy

app/uploaders/avatar_uploader.rb

@@ -25,6 +25,10 @@ class AvatarUploader < GitlabUploader
     self.class.absolute_path(upload)
   end
+  def mounted_as
+    super || 'avatar'
+  end
+
   private
   def dynamic_segment

app/workers/all_queues.yml

@@ -958,7 +958,7 @@
   :resource_boundary: :unknown
   :weight: 1
 - :name: project_export
-  :feature_category: :source_code_management
+  :feature_category: :importers
   :has_external_dependencies: 
   :latency_sensitive: 
   :resource_boundary: :memory

app/workers/project_export_worker.rb

@@ -5,7 +5,7 @@ class ProjectExportWorker
   include ExceptionBacktrace
   sidekiq_options retry: 3
-  feature_category :source_code_management
+  feature_category :importers
   worker_resource_boundary :memory
   def perform(current_user_id, project_id, after_export_strategy = {}, params = {})

changelogs/unreleased/al-196720-remove-destroy_user_associations_in_batches-ff.yml

+---
+title: Destroy user associations in batches like we do with projects
+merge_request: 24641
+author:
+type: performance

changelogs/unreleased/fj-fix-bug-rendering-snippet-blob-markdown.yml

+---
+title: Fix bug rendering BlobType markdown data
+merge_request: 24960
+author:
+type: fixed

changelogs/unreleased/merged-branch-names-cache-v2.yml

+---
+title: Improvement to merged_branch_names cache
+merge_request: 24504
+author:
+type: performance

changelogs/unreleased/tc-fix-38088.yml

+---
+title: Ensure a valid mount_point is used by the AvatarUploader
+merge_request: 24800
+author:
+type: fixed

changelogs/unreleased/turn-on-merged-branch-cache-flag.yml

+---
+title: Cache repository merged branch names by default
+merge_request: 24986
+author:
+type: performance

doc/api/graphql/reference/gitlab_schema.graphql

@@ -6757,11 +6757,6 @@ type SnippetBlob {
   """
   binary: Boolean!
-  """
-  Blob highlighted data
-  """
-  highlightedData: String
-
   """
   Blob mode
   """
@@ -6780,13 +6775,18 @@ type SnippetBlob {
   """
   Blob plain highlighted data
   """
-  plainHighlightedData: String
+  plainData: String
   """
   Blob raw content endpoint path
   """
   rawPath: String!
+  """
+  Blob highlighted data
+  """
+  richData: String
+
   """
   Blob content rich viewer
   """

doc/api/graphql/reference/gitlab_schema.json

@@ -7556,20 +7556,6 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
-            {
-              "name": "highlightedData",
-              "description": "Blob highlighted data",
-              "args": [
-
-              ],
-              "type": {
-                "kind": "SCALAR",
-                "name": "String",
-                "ofType": null
-              },
-              "isDeprecated": false,
-              "deprecationReason": null
-            },
             {
               "name": "mode",
               "description": "Blob mode",
@@ -7613,7 +7599,7 @@
               "deprecationReason": null
             },
             {
-              "name": "plainHighlightedData",
+              "name": "plainData",
               "description": "Blob plain highlighted data",
               "args": [
@@ -7644,6 +7630,20 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
+            {
+              "name": "richData",
+              "description": "Blob highlighted data",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "String",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
             {
               "name": "richViewer",
               "description": "Blob content rich viewer",

doc/api/graphql/reference/index.md

@@ -1067,12 +1067,12 @@ Represents the snippet blob
 | Name  | Type  | Description |
 | ---   |  ---- | ----------  |
 | `binary` | Boolean! | Shows whether the blob is binary |
-| `highlightedData` | String | Blob highlighted data |
 | `mode` | String | Blob mode |
 | `name` | String | Blob name |
 | `path` | String | Blob path |
-| `plainHighlightedData` | String | Blob plain highlighted data |
+| `plainData` | String | Blob plain highlighted data |
 | `rawPath` | String! | Blob raw content endpoint path |
+| `richData` | String | Blob highlighted data |
 | `richViewer` | SnippetBlobViewer | Blob content rich viewer |
 | `simpleViewer` | SnippetBlobViewer! | Blob content simple viewer |
 | `size` | Int! | Blob size |

doc/development/i18n/proofreader.md

@@ -7,6 +7,8 @@ are very appreciative of the work done by translators and proofreaders!
 - Albanian
   - Proofreaders needed.
+- Amharic
+  - Tsegaselassie Tadesse - [GitLab](https://gitlab.com/tsega), [Crowdin](https://crowdin.com/profile/tsegaselassi/activity)
 - Arabic
   - Proofreaders needed.
 - Bulgarian

doc/user/project/clusters/kubernetes_pod_logs.md

@@ -65,13 +65,13 @@ you can search the content of your logs via a search bar.
 The search is passed on to Elasticsearch using the [simple_query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html)
 Elasticsearch function, which supports the following operators:
-```
-+ signifies AND operation
-| signifies OR operation
-- negates a single token
-" wraps a number of tokens to signify a phrase for searching
-* at the end of a term signifies a prefix query
-( and ) signify precedence
-~N after a word signifies edit distance (fuzziness)
-~N after a phrase signifies slop amount
-```
+| Operator                   | Description                                                |
+|----------------------------|------------------------------------------------------------|
+|  `\|`                      | An OR operation.                                            |
+| `-`                        | Negates a single token.                                     |
+| `+`                        | An AND operation.                                           |
+| `"`                        | Wraps a number of tokens to signify a phrase for searching. |
+| `*` (at the end of a term) | A prefix query.                                             |
+| `(` and `)`                | Precedence.                                                 |
+| `~N` (after a word)        | Edit distance (fuzziness).                                  |
+| `~N` (after a phrase)      | Slop amount.                                                |

lib/gitlab/redis/boolean.rb

+# frozen_string_literal: true
+
+# A serializer for boolean values being stored in Redis.
+#
+# This is to ensure that booleans are stored in a consistent and
+# testable way when being stored as strings in Redis.
+#
+# Examples:
+#
+#     bool = Gitlab::Redis::Boolean.new(true)
+#     bool.to_s == "_b:1"
+#
+#     Gitlab::Redis::Boolean.encode(true)
+#     => "_b:1"
+#
+#     Gitlab::Redis::Boolean.decode("_b:1")
+#     => true
+#
+#     Gitlab::Redis::Boolean.true?("_b:1")
+#     => true
+#
+#     Gitlab::Redis::Boolean.true?("_b:0")
+#     => false
+
+module Gitlab
+  module Redis
+    class Boolean
+      LABEL = "_b"
+      DELIMITER = ":"
+      TRUE_STR = "1"
+      FALSE_STR = "0"
+
+      BooleanError = Class.new(StandardError)
+      NotABooleanError = Class.new(BooleanError)
+      NotAnEncodedBooleanStringError = Class.new(BooleanError)
+
+      def initialize(value)
+        @value = value
+      end
+
+      # @return [String] the encoded boolean
+      def to_s
+        self.class.encode(@value)
+      end
+
+      class << self
+        # Turn a boolean into a string for storage in Redis
+        #
+        # @param value [Boolean] true or false
+        # @return [String] the encoded boolean
+        # @raise [NotABooleanError] if the value isn't true or false
+        def encode(value)
+          raise NotABooleanError.new(value) unless bool?(value)
+
+          [LABEL, to_string(value)].join(DELIMITER)
+        end
+
+        # Decode a boolean string
+        #
+        # @param value [String] the stored boolean string
+        # @return [Boolean] true or false
+        # @raise [NotAnEncodedBooleanStringError] if the provided value isn't an encoded boolean
+        def decode(value)
+          raise NotAnEncodedBooleanStringError.new(value.class) unless value.is_a?(String)
+
+          label, bool_str = *value.split(DELIMITER, 2)
+
+          raise NotAnEncodedBooleanStringError.new(label) unless label == LABEL
+
+          from_string(bool_str)
+        end
+
+        # Decode a boolean string, then test if it's true
+        #
+        # @param value [String] the stored boolean string
+        # @return [Boolean] is the value true?
+        # @raise [NotAnEncodedBooleanStringError] if the provided value isn't an encoded boolean
+        def true?(encoded_value)
+          decode(encoded_value)
+        end
+
+        # Decode a boolean string, then test if it's false
+        #
+        # @param value [String] the stored boolean string
+        # @return [Boolean] is the value false?
+        # @raise [NotAnEncodedBooleanStringError] if the provided value isn't an encoded boolean
+        def false?(encoded_value)
+          !true?(encoded_value)
+        end
+
+        private
+
+        def bool?(value)
+          [true, false].include?(value)
+        end
+
+        def to_string(bool)
+          bool ? TRUE_STR : FALSE_STR
+        end
+
+        def from_string(str)
+          raise NotAnEncodedBooleanStringError.new(str) unless [TRUE_STR, FALSE_STR].include?(str)
+
+          str == TRUE_STR
+        end
+      end
+    end
+  end
+end