Add latest changes from gitlab-org/gitlab@master

lib/gitlab/exception_log_formatter.rb

@@ -13,7 +13,7 @@ module Gitlab
       )
       if exception.backtrace
-        payload['exception.backtrace'] = Gitlab::Profiler.clean_backtrace(exception.backtrace)
+        payload['exception.backtrace'] = Gitlab::BacktraceCleaner.clean_backtrace(exception.backtrace)
       end
     end
   end

lib/gitlab/git/rugged_impl/use_rugged.rb

@@ -27,7 +27,7 @@ module Gitlab
                 feature: method_name,
                 args: args,
                 duration: duration,
-                backtrace: Gitlab::Profiler.clean_backtrace(caller))
+                backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller))
             end
             result

lib/gitlab/gitaly_client.rb

@@ -179,7 +179,7 @@ module Gitlab
       self.query_time += duration
       if Gitlab::PerformanceBar.enabled_for_request?
         add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc,
-                         backtrace: Gitlab::Profiler.clean_backtrace(caller))
+                         backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller))
       end
     end
@@ -438,7 +438,7 @@ module Gitlab
     def self.count_stack
       return unless Gitlab::SafeRequestStore.active?
-      stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n")
+      stack_string = Gitlab::BacktraceCleaner.clean_backtrace(caller).drop(1).join("\n")
       Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new

lib/gitlab/profiler.rb

@@ -107,7 +107,7 @@ module Gitlab
             super
-            Gitlab::Profiler.clean_backtrace(caller).each do |caller_line|
+            Gitlab::BacktraceCleaner.clean_backtrace(caller).each do |caller_line|
               stripped_caller_line = caller_line.sub("#{Rails.root}/", '')
               super("  ↳ #{stripped_caller_line}")
@@ -117,14 +117,6 @@ module Gitlab
       end
     end
-    def self.clean_backtrace(backtrace)
-      return unless backtrace
-
-      Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line|
-        line.match(Regexp.union(IGNORE_BACKTRACES))
-      end
-    end
-
     def self.with_custom_logger(logger)
       original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging
       original_activerecord_logger = ActiveRecord::Base.logger

lib/gitlab/repo_path.rb

@@ -32,9 +32,12 @@ module Gitlab
     def self.find_project(project_path)
       project = Project.find_by_full_path(project_path, follow_redirects: true)
-      was_redirected = project && project.full_path.casecmp(project_path) != 0
-      [project, was_redirected]
+      [project, redirected?(project, project_path)]
+    end
+
+    def self.redirected?(project, project_path)
+      project && project.full_path.casecmp(project_path) != 0
     end
   end
 end

lib/gitlab/repository_cache.rb

@@ -7,7 +7,8 @@ module Gitlab
     def initialize(repository, extra_namespace: nil, backend: Rails.cache)
       @repository = repository
-      @namespace = "#{repository.full_path}:#{repository.project.id}"
+      @namespace = "#{repository.full_path}"
+      @namespace += ":#{repository.project.id}" if repository.project
       @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace
       @backend = backend
     end

lib/gitlab/repository_set_cache.rb

@@ -7,7 +7,8 @@ module Gitlab
     def initialize(repository, extra_namespace: nil, expires_in: 2.weeks)
       @repository = repository
-      @namespace = "#{repository.full_path}:#{repository.project.id}"
+      @namespace = "#{repository.full_path}"
+      @namespace += ":#{repository.project.id}" if repository.project
       @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace
       @expires_in = expires_in
     end

lib/gitlab/sidekiq_logging/exception_handler.rb

@@ -18,7 +18,7 @@ module Gitlab
           data.merge!(job_data) if job_data.present?
         end
-        data[:error_backtrace] = Gitlab::Profiler.clean_backtrace(job_exception.backtrace) if job_exception.backtrace.present?
+        data[:error_backtrace] = Gitlab::BacktraceCleaner.clean_backtrace(job_exception.backtrace) if job_exception.backtrace.present?
         Sidekiq.logger.warn(data)
       end

lib/gitlab/workhorse.rb

@@ -22,18 +22,16 @@ module Gitlab
       def git_http_ok(repository, repo_type, user, action, show_all_refs: false)
         raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s)
-        project = repository.project
-
         attrs = {
           GL_ID: Gitlab::GlId.gl_id(user),
-          GL_REPOSITORY: repo_type.identifier_for_subject(project),
+          GL_REPOSITORY: repo_type.identifier_for_subject(repository.project),
           GL_USERNAME: user&.username,
           ShowAllRefs: show_all_refs,
           Repository: repository.gitaly_repository.to_h,
           GitConfigOptions: [],
           GitalyServer: {
-            address: Gitlab::GitalyClient.address(project.repository_storage),
-            token: Gitlab::GitalyClient.token(project.repository_storage),
+            address: Gitlab::GitalyClient.address(repository.storage),
+            token: Gitlab::GitalyClient.token(repository.storage),
             features: Feature::Gitaly.server_feature_flags
           }
         }

lib/peek/views/active_record.rb

@@ -32,7 +32,7 @@ module Peek
               detail_store << {
                 duration: finish - start,
                 sql: data[:sql].strip,
-                backtrace: Gitlab::Profiler.clean_backtrace(caller)
+                backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)
               }
             end
           end

lib/peek/views/redis_detailed.rb

@@ -23,7 +23,7 @@ module Gitlab
         detail_store << {
           cmd: args.first,
           duration: duration,
-          backtrace: ::Gitlab::Profiler.clean_backtrace(caller)
+          backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
         }
       end

qa/qa/resource/runner.rb

@@ -54,7 +54,7 @@ module QA
         @id = this_runner[:id]
         super
-
+      ensure
         Service::DockerRun::GitlabRunner.new(name).remove!
       end

qa/qa/resource/sandbox.rb

@@ -41,6 +41,14 @@ module QA
         resource_web_url(api_get)
       rescue ResourceNotFoundError
         super
+
+        # If the group was just created the runners token might not be
+        # available via the API immediately.
+        Support::Retrier.retry_on_exception(sleep_interval: 5) do
+          resource = resource_web_url(api_get)
+          populate(:runners_token)
+          resource
+        end
       end
       def api_get_path

spec/controllers/projects/git_http_controller_spec.rb

@@ -3,10 +3,17 @@
 require 'spec_helper'
 describe Projects::GitHttpController do
+  let_it_be(:project) { create(:project, :public, :repository) }
+  let(:project_params) do
+    {
+      namespace_id: project.namespace.to_param,
+      project_id: project.path + '.git'
+    }
+  end
+  let(:params) { project_params }
+
   describe 'HEAD #info_refs' do
     it 'returns 403' do
-      project = create(:project, :public, :repository)
-
       head :info_refs, params: { namespace_id: project.namespace.to_param, project_id: project.path + '.git' }
       expect(response.status).to eq(403)
@@ -14,18 +21,17 @@ describe Projects::GitHttpController do
   end
   describe 'GET #info_refs' do
+    let(:params) { project_params.merge(service: 'git-upload-pack') }
+
     it 'returns 401 for unauthenticated requests to public repositories when http protocol is disabled' do
       stub_application_setting(enabled_git_access_protocol: 'ssh')
-      project = create(:project, :public, :repository)
-      get :info_refs, params: { service: 'git-upload-pack', namespace_id: project.namespace.to_param, project_id: project.path + '.git' }
+      get :info_refs, params: params
       expect(response.status).to eq(401)
     end
     context 'with exceptions' do
-      let(:project) { create(:project, :public, :repository) }
-
       before do
         allow(controller).to receive(:verify_workhorse_api!).and_return(true)
       end
@@ -33,7 +39,7 @@ describe Projects::GitHttpController do
       it 'returns 503 with GRPC Unavailable' do
         allow(controller).to receive(:access_check).and_raise(GRPC::Unavailable)
-        get :info_refs, params: { service: 'git-upload-pack', namespace_id: project.namespace.to_param, project_id: project.path + '.git' }
+        get :info_refs, params: params
         expect(response.status).to eq(503)
       end
@@ -41,11 +47,27 @@ describe Projects::GitHttpController do
       it 'returns 503 with timeout error' do
         allow(controller).to receive(:access_check).and_raise(Gitlab::GitAccess::TimeoutError)
-        get :info_refs, params: { service: 'git-upload-pack', namespace_id: project.namespace.to_param, project_id: project.path + '.git' }
+        get :info_refs, params: params
         expect(response.status).to eq(503)
         expect(response.body).to eq 'Gitlab::GitAccess::TimeoutError'
       end
     end
   end
+
+  describe 'POST #git_upload_pack' do
+    before do
+      allow(controller).to receive(:authenticate_user).and_return(true)
+      allow(controller).to receive(:verify_workhorse_api!).and_return(true)
+      allow(controller).to receive(:access_check).and_return(nil)
+    end
+
+    after do
+      post :git_upload_pack, params: params
+    end
+
+    it 'updates project statistics' do
+      expect(ProjectDailyStatisticsWorker).to receive(:perform_async)
+    end
+  end
 end

spec/fixtures/api/schemas/public_api/v4/issue.json

@@ -84,6 +84,11 @@
       "total_time_spent": { "type": "integer" },
       "human_time_estimate": { "type": ["string", "null"] },
       "human_total_time_spent": { "type": ["string", "null"] }
+    },
+    "references": {
+      "short": {"type": "string"},
+      "relative": {"type": "string"},
+      "full": {"type": "string"}
     }
   },
   "required": [

spec/fixtures/api/schemas/public_api/v4/merge_request.json

@@ -113,7 +113,12 @@
         "human_total_time_spent": { "type": ["string", "null"] }
       },
       "allow_collaboration": { "type": ["boolean", "null"] },
-      "allow_maintainer_to_push": { "type": ["boolean", "null"] }
+      "allow_maintainer_to_push": { "type": ["boolean", "null"] },
+      "references": {
+        "short": {"type": "string"},
+        "relative": {"type": "string"},
+        "full": {"type": "string"}
+      }
     },
     "required": [
       "id", "iid", "project_id", "title", "description",

spec/initializers/lograge_spec.rb

@@ -112,7 +112,7 @@ describe 'lograge', type: :request do
         expect(log_data['exception.class']).to eq('RuntimeError')
         expect(log_data['exception.message']).to eq('bad request')
-        expect(log_data['exception.backtrace']).to eq(Gitlab::Profiler.clean_backtrace(backtrace))
+        expect(log_data['exception.backtrace']).to eq(Gitlab::BacktraceCleaner.clean_backtrace(backtrace))
       end
     end
   end

spec/lib/gitlab/backtrace_cleaner_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BacktraceCleaner do
+  describe '.clean_backtrace' do
+    it 'uses the Rails backtrace cleaner' do
+      backtrace = []
+
+      expect(Rails.backtrace_cleaner).to receive(:clean).with(backtrace)
+
+      described_class.clean_backtrace(backtrace)
+    end
+
+    it 'removes lines from IGNORE_BACKTRACES' do
+      backtrace = [
+        "lib/gitlab/gitaly_client.rb:294:in `block (2 levels) in migrate'",
+        "lib/gitlab/gitaly_client.rb:331:in `allow_n_plus_1_calls'",
+        "lib/gitlab/gitaly_client.rb:280:in `block in migrate'",
+        "lib/gitlab/metrics/influx_db.rb:103:in `measure'",
+        "lib/gitlab/gitaly_client.rb:278:in `migrate'",
+        "lib/gitlab/git/repository.rb:1451:in `gitaly_migrate'",
+        "lib/gitlab/git/commit.rb:66:in `find'",
+        "app/models/repository.rb:1047:in `find_commit'",
+        "lib/gitlab/metrics/instrumentation.rb:159:in `block in find_commit'",
+        "lib/gitlab/metrics/method_call.rb:36:in `measure'",
+        "lib/gitlab/metrics/instrumentation.rb:159:in `find_commit'",
+        "app/models/repository.rb:113:in `commit'",
+        "lib/gitlab/i18n.rb:50:in `with_locale'",
+        "lib/gitlab/middleware/multipart.rb:95:in `call'",
+        "lib/gitlab/request_profiler/middleware.rb:14:in `call'",
+        "ee/lib/gitlab/database/load_balancing/rack_middleware.rb:37:in `call'",
+        "ee/lib/gitlab/jira/middleware.rb:15:in `call'"
+      ]
+
+      expect(described_class.clean_backtrace(backtrace))
+        .to eq([
+                 "lib/gitlab/gitaly_client.rb:294:in `block (2 levels) in migrate'",
+                 "lib/gitlab/gitaly_client.rb:331:in `allow_n_plus_1_calls'",
+                 "lib/gitlab/gitaly_client.rb:280:in `block in migrate'",
+                 "lib/gitlab/gitaly_client.rb:278:in `migrate'",
+                 "lib/gitlab/git/repository.rb:1451:in `gitaly_migrate'",
+                 "lib/gitlab/git/commit.rb:66:in `find'",
+                 "app/models/repository.rb:1047:in `find_commit'",
+                 "app/models/repository.rb:113:in `commit'",
+                 "ee/lib/gitlab/jira/middleware.rb:15:in `call'"
+               ])
+    end
+  end
+end

spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb

@@ -39,7 +39,7 @@ describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do
         before do
           current_backtrace = caller
           allow(exception).to receive(:backtrace).and_return(current_backtrace)
-          expected['exception.backtrace'] = Gitlab::Profiler.clean_backtrace(current_backtrace)
+          expected['exception.backtrace'] = Gitlab::BacktraceCleaner.clean_backtrace(current_backtrace)
         end
         it 'includes the backtrace' do

spec/lib/gitlab/profiler_spec.rb

@@ -120,51 +120,6 @@ describe Gitlab::Profiler do
     end
   end
-  describe '.clean_backtrace' do
-    it 'uses the Rails backtrace cleaner' do
-      backtrace = []
-
-      expect(Rails.backtrace_cleaner).to receive(:clean).with(backtrace)
-
-      described_class.clean_backtrace(backtrace)
-    end
-
-    it 'removes lines from IGNORE_BACKTRACES' do
-      backtrace = [
-        "lib/gitlab/gitaly_client.rb:294:in `block (2 levels) in migrate'",
-        "lib/gitlab/gitaly_client.rb:331:in `allow_n_plus_1_calls'",
-        "lib/gitlab/gitaly_client.rb:280:in `block in migrate'",
-        "lib/gitlab/metrics/influx_db.rb:103:in `measure'",
-        "lib/gitlab/gitaly_client.rb:278:in `migrate'",
-        "lib/gitlab/git/repository.rb:1451:in `gitaly_migrate'",
-        "lib/gitlab/git/commit.rb:66:in `find'",
-        "app/models/repository.rb:1047:in `find_commit'",
-        "lib/gitlab/metrics/instrumentation.rb:159:in `block in find_commit'",
-        "lib/gitlab/metrics/method_call.rb:36:in `measure'",
-        "lib/gitlab/metrics/instrumentation.rb:159:in `find_commit'",
-        "app/models/repository.rb:113:in `commit'",
-        "lib/gitlab/i18n.rb:50:in `with_locale'",
-        "lib/gitlab/middleware/multipart.rb:95:in `call'",
-        "lib/gitlab/request_profiler/middleware.rb:14:in `call'",
-        "ee/lib/gitlab/database/load_balancing/rack_middleware.rb:37:in `call'",
-        "ee/lib/gitlab/jira/middleware.rb:15:in `call'"
-      ]
-
-      expect(described_class.clean_backtrace(backtrace))
-        .to eq([
-                 "lib/gitlab/gitaly_client.rb:294:in `block (2 levels) in migrate'",
-                 "lib/gitlab/gitaly_client.rb:331:in `allow_n_plus_1_calls'",
-                 "lib/gitlab/gitaly_client.rb:280:in `block in migrate'",
-                 "lib/gitlab/gitaly_client.rb:278:in `migrate'",
-                 "lib/gitlab/git/repository.rb:1451:in `gitaly_migrate'",
-                 "lib/gitlab/git/commit.rb:66:in `find'",
-                 "app/models/repository.rb:1047:in `find_commit'",
-                 "app/models/repository.rb:113:in `commit'",
-                 "ee/lib/gitlab/jira/middleware.rb:15:in `call'"
-               ])
-    end
-  end
-
   describe '.with_custom_logger' do
     context 'when the logger is set' do
       it 'uses the replacement logger for the duration of the block' do