Add latest changes from gitlab-org/gitlab@master

doc/user/packages/npm_registry/index.md

@@ -5,7 +5,7 @@
 With the GitLab NPM Registry, every
 project can have its own space to store NPM packages.
-![GitLab NPM Registry](img/npm_package_view.png)
+![GitLab NPM Registry](img/npm_package_view_v12_5.png)
 NOTE: **Note:**
 Only [scoped](https://docs.npmjs.com/misc/scope) packages are supported.
@@ -42,6 +42,20 @@ it is not possible due to a naming collision. For example:
 | `gitlab-org/gitlab`    | `@gitlab-org/gitlab`    | Yes       |
 | `gitlab-org/gitlab`    | `@foo/bar`              | No        |
+The regex that is used for naming is validating all package names from all package managers:
+
+```
+/\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z/
+```
+
+It allows for capital letters, while NPM does not, and allows for almost all of the
+characters NPM allows with a few exceptions (`~` is not allowed).
+
+NOTE: **Note:** Capital letters are needed because the scope is required to be
+identical to the top level namespace of the project. So, for example, if your
+project path is `My-Group/project-foo`, your package must be named `@My-Group/any-package-name`.
+`@my-group/any-package-name` will not work.
+
 CAUTION: **When updating the path of a user/group or transferring a (sub)group/project:**
 If you update the root namespace of a project with NPM packages, your changes will be rejected. To be allowed to do that, make sure to remove any NPM package first. Don't forget to update your `.npmrc` files to follow the above naming convention and run `npm publish` if necessary.

doc/user/project/integrations/webhooks.md

@@ -1285,7 +1285,7 @@ Markdown features, like link labels.
 ## Testing webhooks
-You can trigger the webhook manually. Sample data from the project will be used.Sample data will take from the project.
+You can trigger the webhook manually. Sample data from the project will be used. Sample data will take from the project.
 > For example: for triggering `Push Events` your project should have at least one commit.
 ![Webhook testing](img/webhook_testing.png)

lib/api/project_export.rb

@@ -2,6 +2,15 @@
 module API
   class ProjectExport < Grape::API
+    helpers do
+      def throttled?(action)
+        rate_limiter.throttled?(action, scope: [current_user, action, user_project])
+      end
+
+      def rate_limiter
+        ::Gitlab::ApplicationRateLimiter
+      end
+    end
     before do
       not_found! unless Gitlab::CurrentSettings.project_export_enabled?
       authorize_admin_project
@@ -23,6 +32,10 @@ module API
         detail 'This feature was introduced in GitLab 10.6.'
       end
       get ':id/export/download' do
+        if throttled?(:project_download_export)
+          render_api_error!({ error: 'This endpoint has been requested too many times. Try again later.' }, 429)
+        end
+
         if user_project.export_file_exists?
           present_carrierwave_file!(user_project.export_file)
         else
@@ -41,6 +54,10 @@ module API
         end
       end
       post ':id/export' do
+        if throttled?(:project_export)
+          render_api_error!({ error: 'This endpoint has been requested too many times. Try again later.' }, 429)
+        end
+
         project_export_params = declared_params(include_missing: false)
         after_export_params = project_export_params.delete(:upload) || {}

lib/gitlab/action_rate_limiter.rb

-# frozen_string_literal: true
-
-module Gitlab
-  # This class implements a simple rate limiter that can be used to throttle
-  # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at
-  # the middleware level, this can be used at the controller level.
-  class ActionRateLimiter
-    TIME_TO_EXPIRE = 60 # 1 min
-
-    attr_accessor :action, :expiry_time
-
-    def initialize(action:, expiry_time: TIME_TO_EXPIRE)
-      @action = action
-      @expiry_time = expiry_time
-    end
-
-    # Increments the given cache key and increments the value by 1 with the
-    # given expiration time. Returns the incremented value.
-    #
-    # key - An array of ActiveRecord instances
-    def increment(key)
-      value = 0
-
-      Gitlab::Redis::Cache.with do |redis|
-        cache_key = action_key(key)
-        value = redis.incr(cache_key)
-        redis.expire(cache_key, expiry_time) if value == 1
-      end
-
-      value
-    end
-
-    # Increments the given key and returns true if the action should
-    # be throttled.
-    #
-    # key - An array of ActiveRecord instances or strings
-    # threshold_value - The maximum number of times this action should occur in the given time interval. If number is zero is considered disabled.
-    def throttled?(key, threshold_value)
-      threshold_value > 0 &&
-        self.increment(key) > threshold_value
-    end
-
-    # Logs request into auth.log
-    #
-    # request - Web request to be logged
-    # type - A symbol key that represents the request.
-    # current_user - Current user of the request, it can be nil.
-    def log_request(request, type, current_user)
-      request_information = {
-        message: 'Action_Rate_Limiter_Request',
-        env: type,
-        remote_ip: request.ip,
-        request_method: request.request_method,
-        path: request.fullpath
-      }
-
-      if current_user
-        request_information.merge!({
-          user_id: current_user.id,
-          username: current_user.username
-        })
-      end
-
-      Gitlab::AuthLogger.error(request_information)
-    end
-
-    private
-
-    def action_key(key)
-      serialized = key.map do |obj|
-        if obj.is_a?(String)
-          "#{obj}"
-        else
-          "#{obj.class.model_name.to_s.underscore}:#{obj.id}"
-        end
-      end.join(":")
-
-      "action_rate_limiter:#{action}:#{serialized}"
-    end
-  end
-end

lib/gitlab/application_rate_limiter.rb

+# frozen_string_literal: true
+
+module Gitlab
+  # This class implements a simple rate limiter that can be used to throttle
+  # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at
+  # the middleware level, this can be used at the controller or API level.
+  #
+  # @example
+  #  if Gitlab::ApplicationRateLimiter.throttled?(:project_export, scope: [@project, @current_user])
+  #   flash[:alert] = 'error!'
+  #   redirect_to(edit_project_path(@project), status: :too_many_requests)
+  # end
+  class ApplicationRateLimiter
+    class << self
+      # Application rate limits
+      #
+      # Threshold value can be either an Integer or a Proc
+      # in order to not evaluate it's value every time this method is called
+      # and only do that when it's needed.
+      def rate_limits
+        {
+          project_export:              { threshold: 1, interval: 5.minutes },
+          project_download_export:     { threshold: 10, interval: 10.minutes },
+          project_generate_new_export: { threshold: 1, interval: 5.minutes },
+          play_pipeline_schedule:      { threshold: 1, interval: 1.minute },
+          show_raw_controller:         { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute }
+        }.freeze
+      end
+
+      # Increments the given key and returns true if the action should
+      # be throttled.
+      #
+      # @param key [Symbol] Key attribute registered in `.rate_limits`
+      # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project)
+      # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits`
+      # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits`
+      #
+      # @return [Boolean] Whether or not a request should be throttled
+      def throttled?(key, scope: nil, interval: nil, threshold: nil)
+        return unless rate_limits[key]
+
+        threshold_value = threshold || threshold(key)
+
+        threshold_value > 0 &&
+          increment(key, scope, interval) > threshold_value
+      end
+
+      # Increments the given cache key and increments the value by 1 with the
+      # expiration interval defined in `.rate_limits`.
+      #
+      # @param key [Symbol] Key attribute registered in `.rate_limits`
+      # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project)
+      # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits`
+      #
+      # @return [Integer] incremented value
+      def increment(key, scope, interval = nil)
+        value = 0
+        interval_value = interval || interval(key)
+
+        Gitlab::Redis::Cache.with do |redis|
+          cache_key = action_key(key, scope)
+          value     = redis.incr(cache_key)
+          redis.expire(cache_key, interval_value) if value == 1
+        end
+
+        value
+      end
+
+      # Logs request using provided logger
+      #
+      # @param request [Http::Request] - Web request to be logged
+      # @param type [Symbol] A symbol key that represents the request
+      # @param current_user [User] Current user of the request, it can be nil
+      # @param logger [Logger] Logger to log request to a specific log file. Defaults to Gitlab::AuthLogger
+      def log_request(request, type, current_user, logger = Gitlab::AuthLogger)
+        request_information = {
+          message:        'Application_Rate_Limiter_Request',
+          env:            type,
+          remote_ip:      request.ip,
+          request_method: request.request_method,
+          path:           request.fullpath
+        }
+
+        if current_user
+          request_information.merge!({
+                                       user_id:  current_user.id,
+                                       username: current_user.username
+                                     })
+        end
+
+        logger.error(request_information)
+      end
+
+      private
+
+      def threshold(key)
+        value = rate_limit_value_by_key(key, :threshold)
+
+        return value.call if value.is_a?(Proc)
+
+        value.to_i
+      end
+
+      def interval(key)
+        rate_limit_value_by_key(key, :interval).to_i
+      end
+
+      def rate_limit_value_by_key(key, setting)
+        action = rate_limits[key]
+
+        action[setting] if action
+      end
+
+      def action_key(key, scope)
+        composed_key = [key, scope].flatten.compact
+
+        serialized = composed_key.map do |obj|
+          if obj.is_a?(String) || obj.is_a?(Symbol)
+            "#{obj}"
+          else
+            "#{obj.class.model_name.to_s.underscore}:#{obj.id}"
+          end
+        end.join(":")
+
+        "application_rate_limiter:#{serialized}"
+      end
+    end
+  end
+end

lib/gitlab/gon_helper.rb

@@ -42,6 +42,7 @@ module Gitlab
       # Initialize gon.features with any flags that should be
       # made globally available to the frontend
       push_frontend_feature_flag(:suppress_ajax_navigation_errors, default_enabled: true)
+      push_frontend_feature_flag(:snippets_vue, default_enabled: false)
     end
     # Exposes the state of a feature flag to the frontend code.

locale/gitlab.pot

@@ -17898,6 +17898,9 @@ msgstr ""
 msgid "This domain is not verified. You will need to verify ownership before access is enabled."
 msgstr ""
+msgid "This endpoint has been requested too many times. Try again later."
+msgstr ""
+
 msgid "This environment has no deployments yet."
 msgstr ""

spec/controllers/projects/raw_controller_spec.rb

@@ -65,7 +65,7 @@ describe Projects::RawController do
       it 'logs the event on auth.log' do
         attributes = {
-          message: 'Action_Rate_Limiter_Request',
+          message: 'Application_Rate_Limiter_Request',
           env: :raw_blob_request_limit,
           remote_ip: '0.0.0.0',
           request_method: 'GET',

spec/controllers/projects_controller_spec.rb

@@ -1055,45 +1055,34 @@ describe ProjectsController do
     end
   end
-  describe '#export' do
+  describe 'project export' do
     before do
       sign_in(user)
       project.add_maintainer(user)
     end
-    context 'when project export is enabled' do
-      it 'returns 302' do
-        get :export, params: { namespace_id: project.namespace, id: project }
-
-        expect(response).to have_gitlab_http_status(302)
-      end
-    end
-
-    context 'when project export is disabled' do
+    shared_examples 'rate limits project export endpoint' do
       before do
-        stub_application_setting(project_export_enabled?: false)
+        allow(::Gitlab::ApplicationRateLimiter)
+          .to receive(:throttled?)
+          .and_return(true)
       end
-      it 'returns 404' do
-        get :export, params: { namespace_id: project.namespace, id: project }
-        expect(response).to have_gitlab_http_status(404)
+      it 'prevents requesting project export' do
+        get action, params: { namespace_id: project.namespace, id: project }
+        expect(flash[:alert]).to eq('This endpoint has been requested too many times. Try again later.')
+        expect(response).to have_gitlab_http_status(302)
       end
     end
-  end
-  describe '#download_export' do
-    before do
-      sign_in(user)
-      project.add_maintainer(user)
-    end
-
-    context 'object storage enabled' do
+    describe '#export' do
+      let(:action) { :export }
       context 'when project export is enabled' do
         it 'returns 302' do
-          get :download_export, params: { namespace_id: project.namespace, id: project }
+          get action, params: { namespace_id: project.namespace, id: project }
           expect(response).to have_gitlab_http_status(302)
         end
@@ -1105,66 +1094,96 @@ describe ProjectsController do
         end
         it 'returns 404' do
-          get :download_export, params: { namespace_id: project.namespace, id: project }
+          get action, params: { namespace_id: project.namespace, id: project }
           expect(response).to have_gitlab_http_status(404)
         end
       end
+
+      context 'when the endpoint receives requests above the limit', :clean_gitlab_redis_cache do
+        include_examples 'rate limits project export endpoint'
+      end
     end
-  end
-  describe '#remove_export' do
-    before do
-      sign_in(user)
-      project.add_maintainer(user)
-    end
-    context 'when project export is enabled' do
-      it 'returns 302' do
-        post :remove_export, params: { namespace_id: project.namespace, id: project }
-        expect(response).to have_gitlab_http_status(302)
-      end
-    end
-    context 'when project export is disabled' do
-      before do
-        stub_application_setting(project_export_enabled?: false)
-      end
-      it 'returns 404' do
-        post :remove_export, params: { namespace_id: project.namespace, id: project }
-        expect(response).to have_gitlab_http_status(404)
+    describe '#download_export' do
+      let(:action) { :download_export }
+      context 'object storage enabled' do
+        context 'when project export is enabled' do
+          it 'returns 302' do
+            get action, params: { namespace_id: project.namespace, id: project }
+            expect(response).to have_gitlab_http_status(302)
+          end
+        end
+        context 'when project export is disabled' do
+          before do
+            stub_application_setting(project_export_enabled?: false)
+          end
+          it 'returns 404' do
+            get action, params: { namespace_id: project.namespace, id: project }
+            expect(response).to have_gitlab_http_status(404)
+          end
+        end
+        context 'when the endpoint receives requests above the limit', :clean_gitlab_redis_cache do
+          include_examples 'rate limits project export endpoint'
+        end
       end
     end
-  end
-  describe '#generate_new_export' do
-    before do
-      sign_in(user)
-      project.add_maintainer(user)
-    end
-    context 'when project export is enabled' do
-      it 'returns 302' do
-        post :generate_new_export, params: { namespace_id: project.namespace, id: project }
-        expect(response).to have_gitlab_http_status(302)
+    describe '#remove_export' do
+      let(:action) { :remove_export }
+      context 'when project export is enabled' do
+        it 'returns 302' do
+          post action, params: { namespace_id: project.namespace, id: project }
+          expect(response).to have_gitlab_http_status(302)
+        end
+      end
+      context 'when project export is disabled' do
+        before do
+          stub_application_setting(project_export_enabled?: false)
+        end
+
+        it 'returns 404' do
+          post action, params: { namespace_id: project.namespace, id: project }
+
+          expect(response).to have_gitlab_http_status(404)
+        end
       end
     end
-    context 'when project export is disabled' do
-      before do
-        stub_application_setting(project_export_enabled?: false)
+    describe '#generate_new_export' do
+      let(:action) { :generate_new_export }
+
+      context 'when project export is enabled' do
+        it 'returns 302' do
+          post action, params: { namespace_id: project.namespace, id: project }
+
+          expect(response).to have_gitlab_http_status(302)
+        end
       end
-      it 'returns 404' do
-        post :generate_new_export, params: { namespace_id: project.namespace, id: project }
-        expect(response).to have_gitlab_http_status(404)
+      context 'when project export is disabled' do
+        before do
+          stub_application_setting(project_export_enabled?: false)
+        end
+        it 'returns 404' do
+          post action, params: { namespace_id: project.namespace, id: project }
+
+          expect(response).to have_gitlab_http_status(404)
+        end
+      end
+
+      context 'when the endpoint receives requests above the limit', :clean_gitlab_redis_cache do
+        include_examples 'rate limits project export endpoint'
       end
     end
   end

spec/features/snippets/internal_snippet_spec.rb

@@ -5,6 +5,10 @@ require 'spec_helper'
 describe 'Internal Snippets', :js do
   let(:internal_snippet) { create(:personal_snippet, :internal) }
+  before do
+    stub_feature_flags(snippets_vue: false)
+  end
+
   describe 'normal user' do
     before do
       sign_in(create(:user))

spec/features/snippets/notes_on_personal_snippets_spec.rb

@@ -16,6 +16,7 @@ describe 'Comments on personal snippets', :js do
   let!(:other_note) { create(:note_on_personal_snippet) }
   before do
+    stub_feature_flags(snippets_vue: false)
     sign_in user
     visit snippet_path(snippet)

spec/features/snippets/private_snippets_spec.rb

@@ -6,6 +6,7 @@ describe 'Private Snippets', :js do
   let(:user) { create(:user) }
   before do
+    stub_feature_flags(snippets_vue: false)
     sign_in(user)
   end

spec/features/snippets/public_snippets_spec.rb

@@ -3,6 +3,10 @@
 require 'spec_helper'
 describe 'Public Snippets', :js do
+  before do
+    stub_feature_flags(snippets_vue: false)
+  end
+
   it 'Unauthenticated user should see public snippets' do
     public_snippet = create(:personal_snippet, :public)

spec/features/snippets/show_spec.rb

@@ -6,6 +6,10 @@ describe 'Snippet', :js do
   let(:project) { create(:project, :repository) }
   let(:snippet) { create(:personal_snippet, :public, file_name: file_name, content: content) }
+  before do
+    stub_feature_flags(snippets_vue: false)
+  end
+
   context 'Ruby file' do
     let(:file_name) { 'popen.rb' }
     let(:content) { project.repository.blob_at('master', 'files/ruby/popen.rb').data }

spec/features/snippets/spam_snippets_spec.rb

@@ -7,6 +7,7 @@ describe 'User creates snippet', :js do
   before do
     stub_feature_flags(allow_possible_spam: false)
+    stub_feature_flags(snippets_vue: false)
     stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false')
     Gitlab::CurrentSettings.update!(

spec/features/snippets/user_creates_snippet_spec.rb

@@ -8,6 +8,7 @@ describe 'User creates snippet', :js do
   let(:user) { create(:user) }
   before do
+    stub_feature_flags(snippets_vue: false)
     sign_in(user)
     visit new_snippet_path
   end

spec/features/snippets/user_deletes_snippet_spec.rb

@@ -10,6 +10,8 @@ describe 'User deletes snippet' do
   before do
     sign_in(user)
+    stub_feature_flags(snippets_vue: false)
+
     visit snippet_path(snippet)
   end

spec/features/snippets/user_edits_snippet_spec.rb

@@ -12,6 +12,7 @@ describe 'User edits snippet', :js do
   let(:snippet) { create(:personal_snippet, :public, file_name: file_name, content: content, author: user) }
   before do
+    stub_feature_flags(snippets_vue: false)
     sign_in(user)
     visit edit_snippet_path(snippet)

spec/features/snippets_spec.rb

@@ -6,11 +6,38 @@ describe 'Snippets' do
   context 'when the project has snippets' do
     let(:project) { create(:project, :public) }
     let!(:snippets) { create_list(:project_snippet, 2, :public, author: project.owner, project: project) }
+
     before do
       allow(Snippet).to receive(:default_per_page).and_return(1)
-      visit snippets_path(username: project.owner.username)
+
+      visit project_snippets_path(project)
     end
     it_behaves_like 'paginated snippets'
   end
+
+  describe 'rendering engine' do
+    let_it_be(:snippet) { create(:personal_snippet, :public) }
+    let(:snippets_vue_feature_flag_enabled) { true }
+
+    before do
+      stub_feature_flags(snippets_vue: snippets_vue_feature_flag_enabled)
+
+      visit snippet_path(snippet)
+    end
+
+    it 'renders Vue application' do
+      expect(page).to have_selector('#js-snippet-view')
+      expect(page).not_to have_selector('.personal-snippets')
+    end
+
+    context 'when feature flag is disabled' do
+      let(:snippets_vue_feature_flag_enabled) { false }
+
+      it 'renders HAML application and not Vue' do
+        expect(page).not_to have_selector('#js-snippet-view')
+        expect(page).to have_selector('.personal-snippets')
+      end
+    end
+  end
 end