Add latest changes from gitlab-org/gitlab@master

changelogs/unreleased/210332-approximate-counters-are-not-working-on-gitlab-com.yml

+---
+title: Use batch counters instead of approximate counters in usage data
+merge_request: 27218
+author:
+type: performance

changelogs/unreleased/fix-prevent-user-theme-color-api-overwrite.yml

+---
+title: Prevent default overwrite for theme and color ID in user API
+merge_request: 26792
+author: Fabio Huser
+type: fixed

changelogs/unreleased/update-puma-to-4-3-3.yml

+---
+title: Update Puma to 4.3.3
+merge_request: 27232
+author:
+type: security

doc/api/packages.md

@@ -67,6 +67,7 @@ GET /groups/:id/packages
 | `exclude_subgroups` | boolean | false | If the param is included as true, packages from projects from subgroups are not listed. Default is `false`. |
 | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, `type`, or `project_path`. |
 | `sort`    | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. |
+| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm` or `nuget`. (_Introduced in GitLab 12.9_) |
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/packages?exclude_subgroups=true

lib/api/pipeline_schedules.rb

@@ -22,7 +22,7 @@ module API
       get ':id/pipeline_schedules' do
         authorize! :read_pipeline_schedule, user_project
-        schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope])
+        schedules = Ci::PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope])
           .preload([:owner, :last_pipeline])
         present paginate(schedules), with: Entities::PipelineSchedule
       end

lib/api/pipelines.rb

@@ -27,7 +27,7 @@ module API
         optional :username, type: String, desc: 'The username of the user who triggered pipelines'
         optional :updated_before, type: DateTime, desc: 'Return pipelines updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ'
         optional :updated_after, type: DateTime, desc: 'Return pipelines updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ'
-        optional :order_by, type: String, values: PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id',
+        optional :order_by, type: String, values: Ci::PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id',
                             desc: 'Order pipelines'
         optional :sort,     type: String, values: %w[asc desc], default: 'desc',
                             desc: 'Sort pipelines'
@@ -36,7 +36,7 @@ module API
         authorize! :read_pipeline, user_project
         authorize! :read_build, user_project
-        pipelines = PipelinesFinder.new(user_project, current_user, params).execute
+        pipelines = Ci::PipelinesFinder.new(user_project, current_user, params).execute
         present paginate(pipelines), with: Entities::PipelineBasic
       end

lib/api/runners.rb

@@ -115,7 +115,7 @@ module API
       params do
         requires :id, type: Integer, desc: 'The ID of the runner'
         optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES
-        optional :order_by, type: String, desc: 'Order by `id` or not', values: RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS
+        optional :order_by, type: String, desc: 'Order by `id` or not', values: Ci::RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS
         optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort by asc (ascending) or desc (descending)'
         use :pagination
       end
@@ -123,7 +123,7 @@ module API
         runner = get_runner(params[:id])
         authenticate_list_runners_jobs!(runner)
-        jobs = RunnerJobsFinder.new(runner, params).execute
+        jobs = Ci::RunnerJobsFinder.new(runner, params).execute
         present paginate(jobs), with: Entities::JobBasicWithProject
       end

lib/api/users.rb

@@ -52,8 +52,8 @@ module API
           optional :external, type: Boolean, desc: 'Flag indicating the user is an external user'
           # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
           optional :avatar, type: File, desc: 'Avatar image for user' # rubocop:disable Scalability/FileUploads
-          optional :theme_id, type: Integer, default: 1, desc: 'The GitLab theme for the user'
-          optional :color_scheme_id, type: Integer, default: 1, desc: 'The color scheme for the file viewer'
+          optional :theme_id, type: Integer, desc: 'The GitLab theme for the user'
+          optional :color_scheme_id, type: Integer, desc: 'The color scheme for the file viewer'
           optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile'
           all_or_none_of :extern_uid, :provider

lib/gitlab/cache/import/caching.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Cache
+    module Import
+      module Caching
+        # The default timeout of the cache keys.
+        TIMEOUT = 24.hours.to_i
+
+        WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze
+        local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2]
+        local existing = tonumber(redis.call("get", key))
+
+        if existing == nil or value > existing then
+          redis.call("set", key, value)
+          redis.call("expire", key, ttl)
+          return true
+        else
+          return false
+        end
+        EOF
+
+        # Reads a cache key.
+        #
+        # If the key exists and has a non-empty value its TTL is refreshed
+        # automatically.
+        #
+        # raw_key - The cache key to read.
+        # timeout - The new timeout of the key if the key is to be refreshed.
+        def self.read(raw_key, timeout: TIMEOUT)
+          key = cache_key_for(raw_key)
+          value = Redis::Cache.with { |redis| redis.get(key) }
+
+          if value.present?
+            # We refresh the expiration time so frequently used keys stick
+            # around, removing the need for querying the database as much as
+            # possible.
+            #
+            # A key may be empty when we looked up a GitHub user (for example) but
+            # did not find a matching GitLab user. In that case we _don't_ want to
+            # refresh the TTL so we automatically pick up the right data when said
+            # user were to register themselves on the GitLab instance.
+            Redis::Cache.with { |redis| redis.expire(key, timeout) }
+          end
+
+          value
+        end
+
+        # Reads an integer from the cache, or returns nil if no value was found.
+        #
+        # See Caching.read for more information.
+        def self.read_integer(raw_key, timeout: TIMEOUT)
+          value = read(raw_key, timeout: timeout)
+
+          value.to_i if value.present?
+        end
+
+        # Sets a cache key to the given value.
+        #
+        # key - The cache key to write.
+        # value - The value to set.
+        # timeout - The time after which the cache key should expire.
+        def self.write(raw_key, value, timeout: TIMEOUT)
+          key = cache_key_for(raw_key)
+
+          Redis::Cache.with do |redis|
+            redis.set(key, value, ex: timeout)
+          end
+
+          value
+        end
+
+        # Adds a value to a set.
+        #
+        # raw_key - The key of the set to add the value to.
+        # value - The value to add to the set.
+        # timeout - The new timeout of the key.
+        def self.set_add(raw_key, value, timeout: TIMEOUT)
+          key = cache_key_for(raw_key)
+
+          Redis::Cache.with do |redis|
+            redis.multi do |m|
+              m.sadd(key, value)
+              m.expire(key, timeout)
+            end
+          end
+        end
+
+        # Returns true if the given value is present in the set.
+        #
+        # raw_key - The key of the set to check.
+        # value - The value to check for.
+        def self.set_includes?(raw_key, value)
+          key = cache_key_for(raw_key)
+
+          Redis::Cache.with do |redis|
+            redis.sismember(key, value)
+          end
+        end
+
+        # Sets multiple keys to a given value.
+        #
+        # mapping - A Hash mapping the cache keys to their values.
+        # timeout - The time after which the cache key should expire.
+        def self.write_multiple(mapping, timeout: TIMEOUT)
+          Redis::Cache.with do |redis|
+            redis.multi do |multi|
+              mapping.each do |raw_key, value|
+                multi.set(cache_key_for(raw_key), value, ex: timeout)
+              end
+            end
+          end
+        end
+
+        # Sets the expiration time of a key.
+        #
+        # raw_key - The key for which to change the timeout.
+        # timeout - The new timeout.
+        def self.expire(raw_key, timeout)
+          key = cache_key_for(raw_key)
+
+          Redis::Cache.with do |redis|
+            redis.expire(key, timeout)
+          end
+        end
+
+        # Sets a key to the given integer but only if the existing value is
+        # smaller than the given value.
+        #
+        # This method uses a Lua script to ensure the read and write are atomic.
+        #
+        # raw_key - The key to set.
+        # value - The new value for the key.
+        # timeout - The key timeout in seconds.
+        #
+        # Returns true when the key was overwritten, false otherwise.
+        def self.write_if_greater(raw_key, value, timeout: TIMEOUT)
+          key = cache_key_for(raw_key)
+          val = Redis::Cache.with do |redis|
+            redis
+              .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout])
+          end
+
+          val ? true : false
+        end
+
+        def self.cache_key_for(raw_key)
+          "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}"
+        end
+      end
+    end
+  end
+end

lib/gitlab/github_import.rb

@@ -16,7 +16,7 @@ module Gitlab
     def self.ghost_user_id
       key = 'github-import/ghost-user-id'
-      Caching.read_integer(key) || Caching.write(key, User.select(:id).ghost.id)
+      Gitlab::Cache::Import::Caching.read_integer(key) || Gitlab::Cache::Import::Caching.write(key, User.select(:id).ghost.id)
     end
   end
 end

lib/gitlab/github_import/caching.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module GithubImport
-    module Caching
-      # The default timeout of the cache keys.
-      TIMEOUT = 24.hours.to_i
-
-      WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze
-      local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2]
-      local existing = tonumber(redis.call("get", key))
-
-      if existing == nil or value > existing then
-        redis.call("set", key, value)
-        redis.call("expire", key, ttl)
-        return true
-      else
-        return false
-      end
-      EOF
-
-      # Reads a cache key.
-      #
-      # If the key exists and has a non-empty value its TTL is refreshed
-      # automatically.
-      #
-      # raw_key - The cache key to read.
-      # timeout - The new timeout of the key if the key is to be refreshed.
-      def self.read(raw_key, timeout: TIMEOUT)
-        key = cache_key_for(raw_key)
-        value = Redis::Cache.with { |redis| redis.get(key) }
-
-        if value.present?
-          # We refresh the expiration time so frequently used keys stick
-          # around, removing the need for querying the database as much as
-          # possible.
-          #
-          # A key may be empty when we looked up a GitHub user (for example) but
-          # did not find a matching GitLab user. In that case we _don't_ want to
-          # refresh the TTL so we automatically pick up the right data when said
-          # user were to register themselves on the GitLab instance.
-          Redis::Cache.with { |redis| redis.expire(key, timeout) }
-        end
-
-        value
-      end
-
-      # Reads an integer from the cache, or returns nil if no value was found.
-      #
-      # See Caching.read for more information.
-      def self.read_integer(raw_key, timeout: TIMEOUT)
-        value = read(raw_key, timeout: timeout)
-
-        value.to_i if value.present?
-      end
-
-      # Sets a cache key to the given value.
-      #
-      # key - The cache key to write.
-      # value - The value to set.
-      # timeout - The time after which the cache key should expire.
-      def self.write(raw_key, value, timeout: TIMEOUT)
-        key = cache_key_for(raw_key)
-
-        Redis::Cache.with do |redis|
-          redis.set(key, value, ex: timeout)
-        end
-
-        value
-      end
-
-      # Adds a value to a set.
-      #
-      # raw_key - The key of the set to add the value to.
-      # value - The value to add to the set.
-      # timeout - The new timeout of the key.
-      def self.set_add(raw_key, value, timeout: TIMEOUT)
-        key = cache_key_for(raw_key)
-
-        Redis::Cache.with do |redis|
-          redis.multi do |m|
-            m.sadd(key, value)
-            m.expire(key, timeout)
-          end
-        end
-      end
-
-      # Returns true if the given value is present in the set.
-      #
-      # raw_key - The key of the set to check.
-      # value - The value to check for.
-      def self.set_includes?(raw_key, value)
-        key = cache_key_for(raw_key)
-
-        Redis::Cache.with do |redis|
-          redis.sismember(key, value)
-        end
-      end
-
-      # Sets multiple keys to a given value.
-      #
-      # mapping - A Hash mapping the cache keys to their values.
-      # timeout - The time after which the cache key should expire.
-      def self.write_multiple(mapping, timeout: TIMEOUT)
-        Redis::Cache.with do |redis|
-          redis.multi do |multi|
-            mapping.each do |raw_key, value|
-              multi.set(cache_key_for(raw_key), value, ex: timeout)
-            end
-          end
-        end
-      end
-
-      # Sets the expiration time of a key.
-      #
-      # raw_key - The key for which to change the timeout.
-      # timeout - The new timeout.
-      def self.expire(raw_key, timeout)
-        key = cache_key_for(raw_key)
-
-        Redis::Cache.with do |redis|
-          redis.expire(key, timeout)
-        end
-      end
-
-      # Sets a key to the given integer but only if the existing value is
-      # smaller than the given value.
-      #
-      # This method uses a Lua script to ensure the read and write are atomic.
-      #
-      # raw_key - The key to set.
-      # value - The new value for the key.
-      # timeout - The key timeout in seconds.
-      #
-      # Returns true when the key was overwritten, false otherwise.
-      def self.write_if_greater(raw_key, value, timeout: TIMEOUT)
-        key = cache_key_for(raw_key)
-        val = Redis::Cache.with do |redis|
-          redis
-            .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout])
-        end
-
-        val ? true : false
-      end
-
-      def self.cache_key_for(raw_key)
-        "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}"
-      end
-    end
-  end
-end

lib/gitlab/github_import/issuable_finder.rb

@@ -23,7 +23,7 @@ module Gitlab
       #
       # This method will return `nil` if no ID could be found.
       def database_id
-        val = Caching.read(cache_key)
+        val = Gitlab::Cache::Import::Caching.read(cache_key)
         val.to_i if val.present?
       end
@@ -32,7 +32,7 @@ module Gitlab
       #
       # database_id - The ID of the corresponding database row.
       def cache_database_id(database_id)
-        Caching.write(cache_key, database_id)
+        Gitlab::Cache::Import::Caching.write(cache_key, database_id)
       end
       private

lib/gitlab/github_import/label_finder.rb

@@ -15,7 +15,7 @@ module Gitlab
       # Returns the label ID for the given name.
       def id_for(name)
-        Caching.read_integer(cache_key_for(name))
+        Gitlab::Cache::Import::Caching.read_integer(cache_key_for(name))
       end
       # rubocop: disable CodeReuse/ActiveRecord
@@ -27,7 +27,7 @@ module Gitlab
             hash[cache_key_for(name)] = id
           end
-        Caching.write_multiple(mapping)
+        Gitlab::Cache::Import::Caching.write_multiple(mapping)
       end
       # rubocop: enable CodeReuse/ActiveRecord

lib/gitlab/github_import/milestone_finder.rb

@@ -18,7 +18,7 @@ module Gitlab
       def id_for(issuable)
         return unless issuable.milestone_number
-        Caching.read_integer(cache_key_for(issuable.milestone_number))
+        Gitlab::Cache::Import::Caching.read_integer(cache_key_for(issuable.milestone_number))
       end
       # rubocop: disable CodeReuse/ActiveRecord
@@ -30,7 +30,7 @@ module Gitlab
             hash[cache_key_for(iid)] = id
           end
-        Caching.write_multiple(mapping)
+        Gitlab::Cache::Import::Caching.write_multiple(mapping)
       end
       # rubocop: enable CodeReuse/ActiveRecord

lib/gitlab/github_import/page_counter.rb

@@ -19,12 +19,12 @@ module Gitlab
       #
       # Returns true if the page number was overwritten, false otherwise.
       def set(page)
-        Caching.write_if_greater(cache_key, page)
+        Gitlab::Cache::Import::Caching.write_if_greater(cache_key, page)
       end
       # Returns the current value from the cache.
       def current
-        Caching.read_integer(cache_key) || 1
+        Gitlab::Cache::Import::Caching.read_integer(cache_key) || 1
       end
     end
   end

lib/gitlab/github_import/parallel_scheduling.rb

@@ -42,7 +42,7 @@ module Gitlab
         # still scheduling duplicates while. Since all work has already been
         # completed those jobs will just cycle through any remaining pages while
         # not scheduling anything.
-        Caching.expire(already_imported_cache_key, 15.minutes.to_i)
+        Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i)
         retval
       end
@@ -112,14 +112,14 @@ module Gitlab
       def already_imported?(object)
         id = id_for_already_imported_cache(object)
-        Caching.set_includes?(already_imported_cache_key, id)
+        Gitlab::Cache::Import::Caching.set_includes?(already_imported_cache_key, id)
       end
       # Marks the given object as "already imported".
       def mark_as_imported(object)
         id = id_for_already_imported_cache(object)
-        Caching.set_add(already_imported_cache_key, id)
+        Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, id)
       end
       # Returns the ID to use for the cache used for checking if an object has

lib/gitlab/github_import/user_finder.rb

@@ -102,11 +102,11 @@ module Gitlab
       def email_for_github_username(username)
         cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username
-        email = Caching.read(cache_key)
+        email = Gitlab::Cache::Import::Caching.read(cache_key)
         unless email
           user = client.user(username)
-          email = Caching.write(cache_key, user.email) if user
+          email = Gitlab::Cache::Import::Caching.write(cache_key, user.email) if user
         end
         email
@@ -125,7 +125,7 @@ module Gitlab
       def id_for_github_id(id)
         gitlab_id = query_id_for_github_id(id) || nil
-        Caching.write(ID_CACHE_KEY % id, gitlab_id)
+        Gitlab::Cache::Import::Caching.write(ID_CACHE_KEY % id, gitlab_id)
       end
       # Queries and caches the GitLab user ID for a GitHub email, if one was
@@ -133,7 +133,7 @@ module Gitlab
       def id_for_github_email(email)
         gitlab_id = query_id_for_github_email(email) || nil
-        Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id)
+        Gitlab::Cache::Import::Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id)
       end
       # rubocop: disable CodeReuse/ActiveRecord
@@ -155,7 +155,7 @@ module Gitlab
       # 1. A boolean indicating if the key was present or not.
       # 2. The ID as an Integer, or nil in case no ID could be found.
       def read_id_from_cache(key)
-        value = Caching.read(key)
+        value = Gitlab::Cache::Import::Caching.read(key)
         exists = !value.nil?
         number = value.to_i

lib/gitlab/usage_data.rb

@@ -2,7 +2,6 @@
 module Gitlab
   class UsageData
-    APPROXIMATE_COUNT_MODELS = [Label, MergeRequest, Note, Todo].freeze
     BATCH_SIZE = 100
     class << self
@@ -107,10 +106,12 @@ module Gitlab
             suggestions: count(Suggestion),
             todos: count(Todo),
             uploads: count(Upload),
-            web_hooks: count(WebHook)
+            web_hooks: count(WebHook),
+            labels: count(Label),
+            merge_requests: count(MergeRequest),
+            notes: count(Note)
           }.merge(
             services_usage,
-            approximate_counts,
             usage_counters,
             user_preferences_usage,
             ingress_modsecurity_usage
@@ -251,16 +252,6 @@ module Gitlab
         fallback
       end
-      def approximate_counts
-        approx_counts = Gitlab::Database::Count.approximate_counts(APPROXIMATE_COUNT_MODELS)
-
-        APPROXIMATE_COUNT_MODELS.each_with_object({}) do |model, result|
-          key = model.name.underscore.pluralize.to_sym
-
-          result[key] = approx_counts[model] || -1
-        end
-      end
-
       def installation_type
         if Rails.env.production?
           Gitlab::INSTALLATION_TYPE

locale/gitlab.pot

@@ -1948,6 +1948,9 @@ msgstr ""
 msgid "An error occurred while parsing recent searches"
 msgstr ""
+msgid "An error occurred while parsing the file."
+msgstr ""
+
 msgid "An error occurred while removing epics."
 msgstr ""

spec/features/projects/blobs/blob_show_spec.rb

@@ -308,6 +308,48 @@ describe 'File blob', :js do
     end
   end
+  context 'Jupiter Notebook file' do
+    before do
+      project.add_maintainer(project.creator)
+
+      Files::CreateService.new(
+        project,
+        project.creator,
+        start_branch: 'master',
+        branch_name: 'master',
+        commit_message: "Add Jupiter Notebook",
+        file_path: 'files/basic.ipynb',
+        file_content: project.repository.blob_at('add-ipython-files', 'files/ipython/basic.ipynb').data
+      ).execute
+
+      visit_blob('files/basic.ipynb')
+
+      wait_for_requests
+    end
+
+    it 'displays the blob' do
+      aggregate_failures do
+        # shows rendered notebook
+        expect(page).to have_selector('.js-notebook-viewer-mounted')
+
+        # does show a viewer switcher
+        expect(page).to have_selector('.js-blob-viewer-switcher')
+
+        # show a disabled copy button
+        expect(page).to have_selector('.js-copy-blob-source-btn.disabled')
+
+        # shows a raw button
+        expect(page).to have_link('Open raw')
+
+        # shows a download button
+        expect(page).to have_link('Download')
+
+        # shows the rendered notebook
+        expect(page).to have_content('test')
+      end
+    end
+  end
+
   context 'ISO file (stored in LFS)' do
     context 'when LFS is enabled on the project' do
       before do