Add latest changes from gitlab-org/gitlab@master

app/models/project_services/alerts_service_data.rb

+# frozen_string_literal: true
+
+require 'securerandom'
+
+class AlertsServiceData < ApplicationRecord
+  belongs_to :service, class_name: 'AlertsService'
+
+  validates :service, presence: true
+
+  attr_encrypted :token,
+    mode: :per_attribute_iv,
+    key: Settings.attr_encrypted_db_key_base_truncated,
+    algorithm: 'aes-256-gcm'
+end

app/services/projects/alerting/notify_service.rb

+# frozen_string_literal: true
+
+module Projects
+  module Alerting
+    class NotifyService < BaseService
+      include Gitlab::Utils::StrongMemoize
+
+      def execute(token)
+        return forbidden unless alerts_service_activated?
+        return unauthorized unless valid_token?(token)
+
+        process_incident_issues
+
+        ServiceResponse.success
+      rescue Gitlab::Alerting::NotificationPayloadParser::BadPayloadError
+        bad_request
+      end
+
+      private
+
+      delegate :alerts_service, :alerts_service_activated?, to: :project
+
+      def process_incident_issues
+        IncidentManagement::ProcessAlertWorker
+          .perform_async(project.id, parsed_payload)
+      end
+
+      def parsed_payload
+        Gitlab::Alerting::NotificationPayloadParser.call(params.to_h)
+      end
+
+      def valid_token?(token)
+        token == alerts_service.token
+      end
+
+      def bad_request
+        ServiceResponse.error(message: 'Bad Request', http_status: 400)
+      end
+
+      def unauthorized
+        ServiceResponse.error(message: 'Unauthorized', http_status: 401)
+      end
+
+      def forbidden
+        ServiceResponse.error(message: 'Forbidden', http_status: 403)
+      end
+    end
+  end
+end

app/services/projects/container_repository/delete_tags_service.rb

@@ -14,12 +14,25 @@ module Projects
       private
+      # Delete tags by name with a single DELETE request. This is only supported
+      # by the GitLab Container Registry fork. See
+      # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23325 for details.
+      def fast_delete(container_repository, tag_names)
+        deleted_tags = tag_names.select do |name|
+          container_repository.delete_tag_by_name(name)
+        end
+
+        deleted_tags.any? ? success(deleted: deleted_tags) : error('could not delete tags')
+      end
+
       # Replace a tag on the registry with a dummy tag.
       # This is a hack as the registry doesn't support deleting individual
       # tags. This code effectively pushes a dummy image and assigns the tag to it.
       # This way when the tag is deleted only the dummy image is affected.
+      # This is used to preverse compatibility with third-party registries that
+      # don't support fast delete.
       # See https://gitlab.com/gitlab-org/gitlab/issues/15737 for a discussion
-      def smart_delete(container_repository, tag_names)
+      def slow_delete(container_repository, tag_names)
         # generates the blobs for the dummy image
         dummy_manifest = container_repository.client.generate_empty_manifest(container_repository.path)
         return error('could not generate manifest') if dummy_manifest.nil?
@@ -36,6 +49,15 @@ module Projects
         end
       end
+      def smart_delete(container_repository, tag_names)
+        fast_delete_enabled = Feature.enabled?(:container_registry_fast_tag_delete, default_enabled: true)
+        if fast_delete_enabled && container_repository.client.supports_tag_delete?
+          fast_delete(container_repository, tag_names)
+        else
+          slow_delete(container_repository, tag_names)
+        end
+      end
+
       # update the manifests of the tags with the new dummy image
       def replace_tag_manifests(container_repository, dummy_manifest, tag_names)
         deleted_tags = {}

app/services/projects/lsif_data_service.rb

@@ -2,7 +2,8 @@
 module Projects
   class LsifDataService
-    attr_reader :file, :project, :path, :commit_id
+    attr_reader :file, :project, :path, :commit_id,
+      :docs, :doc_ranges, :ranges, :def_refs
     CACHE_EXPIRE_IN = 1.hour
@@ -14,19 +15,18 @@ module Projects
     end
     def execute
-      docs, doc_ranges, ranges =
-        fetch_data.values_at('docs', 'doc_ranges', 'ranges')
-
-      doc_id = doc_id_from(docs)
+      fetch_data!
       doc_ranges[doc_id]&.map do |range_id|
-        line_data, column_data = ranges[range_id]['loc']
+        location, ref_id = ranges[range_id].values_at('loc', 'ref_id')
+        line_data, column_data = location
         {
           start_line: line_data.first,
           end_line: line_data.last,
           start_char: column_data.first,
-          end_char: column_data.last
+          end_char: column_data.last,
+          definition_url: definition_url_for(def_refs[ref_id])
         }
       end
     end
@@ -47,8 +47,17 @@ module Projects
       end
     end
-    def doc_id_from(docs)
-      docs.reduce(nil) do |doc_id, (id, doc_path)|
+    def fetch_data!
+      data = fetch_data
+
+      @docs = data['docs']
+      @doc_ranges = data['doc_ranges']
+      @ranges = data['ranges']
+      @def_refs = data['def_refs']
+    end
+
+    def doc_id
+      @doc_id ||= docs.reduce(nil) do |doc_id, (id, doc_path)|
         next doc_id unless doc_path =~ /#{path}$/
         if doc_id.nil? || docs[doc_id].size > doc_path.size
@@ -58,5 +67,24 @@ module Projects
         doc_id
       end
     end
+
+    def dir_absolute_path
+      @dir_absolute_path ||= docs[doc_id]&.delete_suffix(path)
+    end
+
+    def definition_url_for(ref_id)
+      return unless range = ranges[ref_id]
+
+      def_doc_id, location = range.values_at('doc_id', 'loc')
+      localized_doc_url = docs[def_doc_id].delete_prefix(dir_absolute_path)
+
+      # location is stored as [[start_line, end_line], [start_char, end_char]]
+      start_line = location.first.first
+
+      line_anchor = "L#{start_line + 1}"
+      definition_ref_path = [commit_id, localized_doc_url].join('/')
+
+      Gitlab::Routing.url_helpers.project_blob_path(project, definition_ref_path, anchor: line_anchor)
+    end
   end
 end

app/views/projects/graphs/charts.html.haml

@@ -7,20 +7,7 @@
   %p
     = _("Measured in bytes of code. Excludes generated and vendored code.")
-  .row
-    .col-md-4
-      %ul.bordered-list
-        - @languages.each do |language|
-          %li
-            %span{ style: "color: #{language[:color]}" }
-              = icon('circle')
-             
-            = language[:label]
-            .float-right
-              = language[:value]
-              \%
-    .col-md-8
-      %canvas#languages-chart{ height: 400 }
+  #js-languages-chart{ data: { chart_data: @languages.to_json.html_safe } }
 .repo-charts
   .sub-header-block.border-top
@@ -60,27 +47,18 @@
       %p.slead
         = _("Commits per day of month")
       %div
-        %canvas#month-chart
+        #js-month-chart{ data: { chart_data: @commits_per_month.to_json.html_safe } }
   .row
     .col-md-6
     .col-md-6
       %p.slead
         = _("Commits per weekday")
       %div
-        %canvas#weekday-chart
+        #js-weekday-chart{ data: { chart_data: @commits_per_week_days.to_json.html_safe } }
   .row
     .col-md-6
     .col-md-6
       %p.slead
         = _("Commits per day hour (UTC)")
       %div
-        %canvas#hour-chart
-
--# haml-lint:disable InlineJavaScript
-%script#projectChartData{ type: "application/json" }
-  - projectChartData = {};
-  - projectChartData['hour'] = @commits_per_time
-  - projectChartData['weekDays'] = @commits_per_week_days
-  - projectChartData['month'] = @commits_per_month
-  - projectChartData['languages'] = @languages
-  = projectChartData.to_json.html_safe
+        #js-hour-chart{ data: { chart_data: @commits_per_time.to_json.html_safe } }

changelogs/unreleased/31832-improve-performance-of-the-container-registry-delete-tags-api.yml

+---
+title: Improve performance of the Container Registry delete tags API
+merge_request: 23325
+author:
+type: performance

changelogs/unreleased/35473-webide-hash-in-branch-name.yml

+---
+title: 'WebIDE: Support # in branch names'
+merge_request: 24717
+author:
+type: changed

changelogs/unreleased/improve-background-migration-check.yml

+---
+title: Add more accurate way of counting remaining background migrations before upgrading
+merge_request:
+author:
+type: fixed

config/initializers/rack_timeout.rb

@@ -12,6 +12,8 @@
 if Gitlab::Runtime.puma? && !Rails.env.test?
   require 'rack/timeout/base'
+  Rack::Timeout::Logger.level = Logger::ERROR
+
   Gitlab::Application.configure do |config|
     config.middleware.insert_before(Rack::Runtime, Rack::Timeout,
                                     service_timeout: ENV.fetch('GITLAB_RAILS_RACK_TIMEOUT', 60).to_i,

config/routes/project.rb

@@ -323,6 +323,8 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         end
       end
+      post 'alerts/notify', to: 'alerting/notifications#create'
+
       resources :pipelines, only: [:index, :new, :create, :show, :destroy] do
         collection do
           resource :pipelines_settings, path: 'settings', only: [:show, :update]

doc/administration/geo/replication/updating_the_geo_nodes.md

@@ -11,6 +11,8 @@ Updating Geo nodes involves performing:
 Depending on which version of Geo you are updating to/from, there may be
 different steps.
+- [Updating to GitLab 12.7](version_specific_updates.md#updating-to-gitlab-127)
+- [Updating to GitLab 12.2](version_specific_updates.md#updating-to-gitlab-122)
 - [Updating to GitLab 12.1](version_specific_updates.md#updating-to-gitlab-121)
 - [Updating to GitLab 10.8](version_specific_updates.md#updating-to-gitlab-108)
 - [Updating to GitLab 10.6](version_specific_updates.md#updating-to-gitlab-106)

doc/update/README.md

@@ -116,14 +116,14 @@ following command:
 **For Omnibus installations**
 ```shell
-sudo gitlab-rails runner -e production 'puts Sidekiq::Queue.new("background_migration").size'
+sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining'
 ```
 **For installations from source**
 ```
 cd /home/git/gitlab
-sudo -u git -H bundle exec rails runner -e production 'puts Sidekiq::Queue.new("background_migration").size'
+sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.remaining'
 ```
 ## Upgrading to a new major version

lib/container_registry/client.rb

@@ -6,6 +6,8 @@ require 'digest'
 module ContainerRegistry
   class Client
+    include Gitlab::Utils::StrongMemoize
+
     attr_accessor :uri
     DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json'
@@ -35,10 +37,25 @@ module ContainerRegistry
       response.headers['docker-content-digest'] if response.success?
     end
-    def delete_repository_tag(name, reference)
-      result = faraday.delete("/v2/#{name}/manifests/#{reference}")
-      result.success? || result.status == 404
+    def delete_repository_tag_by_digest(name, reference)
+      delete_if_exists("/v2/#{name}/manifests/#{reference}")
+    end
+    def delete_repository_tag_by_name(name, reference)
+      delete_if_exists("/v2/#{name}/tags/reference/#{reference}")
+    end
+
+    # Check if the registry supports tag deletion. This is only supported by the
+    # GitLab registry fork. The fastest and safest way to check this is to send
+    # an OPTIONS request to /v2/<name>/tags/reference/<tag>, using a random
+    # repository name and tag (the registry won't check if they exist).
+    # Registries that support tag deletion will reply with a 200 OK and include
+    # the DELETE method in the Allow header. Others reply with an 404 Not Found.
+    def supports_tag_delete?
+      strong_memoize(:supports_tag_delete) do
+        response = faraday.run_request(:options, '/v2/name/tags/reference/tag', '', {})
+        response.success? && response.headers['allow']&.include?('DELETE')
+      end
     end
     def upload_raw_blob(path, blob)
@@ -86,9 +103,7 @@ module ContainerRegistry
     end
     def delete_blob(name, digest)
-      result = faraday.delete("/v2/#{name}/blobs/#{digest}")
-
-      result.success? || result.status == 404
+      delete_if_exists("/v2/#{name}/blobs/#{digest}")
     end
     def put_tag(name, reference, manifest)
@@ -163,6 +178,12 @@ module ContainerRegistry
         conn.adapter :net_http
       end
     end
+
+    def delete_if_exists(path)
+      result = faraday.delete(path)
+
+      result.success? || result.status == 404
+    end
   end
 end

lib/container_registry/tag.rb

@@ -118,7 +118,7 @@ module ContainerRegistry
     def unsafe_delete
       return unless digest
-      client.delete_repository_tag(repository.path, digest)
+      client.delete_repository_tag_by_digest(repository.path, digest)
     end
   end
 end

lib/gitlab/background_migration.rb

@@ -58,6 +58,14 @@ module Gitlab
       migration_class_for(class_name).new.perform(*arguments)
     end
+    def self.remaining
+      scheduled = Sidekiq::ScheduledSet.new.count do |job|
+        job.queue == self.queue
+      end
+
+      scheduled + Sidekiq::Queue.new(self.queue).size
+    end
+
     def self.exists?(migration_class, additional_queues = [])
       enqueued = Sidekiq::Queue.new(self.queue)
       scheduled = Sidekiq::ScheduledSet.new

locale/gitlab.pot

@@ -5996,6 +5996,9 @@ msgstr ""
 msgid "Date range cannot exceed %{maxDateRange} days."
 msgstr ""
+msgid "Day of month"
+msgstr ""
+
 msgid "DayTitle|F"
 msgstr ""
@@ -10004,6 +10007,9 @@ msgstr ""
 msgid "Hook was successfully updated."
 msgstr ""
+msgid "Hour (UTC)"
+msgstr ""
+
 msgid "Housekeeping"
 msgstr ""
@@ -12803,6 +12809,9 @@ msgstr ""
 msgid "No, not interested right now"
 msgstr ""
+msgid "No. of commits"
+msgstr ""
+
 msgid "Nobody has starred this repository yet"
 msgstr ""
@@ -13531,6 +13540,9 @@ msgstr ""
 msgid "People without permission will never get a notification."
 msgstr ""
+msgid "Percentage"
+msgstr ""
+
 msgid "Perform advanced options such as changing path, transferring, or removing the group."
 msgstr ""
@@ -20797,6 +20809,9 @@ msgstr ""
 msgid "Used by members to sign in to your group in GitLab"
 msgstr ""
+msgid "Used programming language"
+msgstr ""
+
 msgid "Used to help configure your identity provider"
 msgstr ""
@@ -21486,6 +21501,9 @@ msgstr ""
 msgid "Wednesday"
 msgstr ""
+msgid "Weekday"
+msgstr ""
+
 msgid "Weeks"
 msgstr ""

spec/controllers/projects/alerting/notifications_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::Alerting::NotificationsController do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:environment) { create(:environment, project: project) }
+
+  describe 'POST #create' do
+    let(:service_response) { ServiceResponse.success }
+    let(:notify_service) { instance_double(Projects::Alerting::NotifyService, execute: service_response) }
+
+    around do |example|
+      ForgeryProtection.with_forgery_protection { example.run }
+    end
+
+    before do
+      allow(Projects::Alerting::NotifyService).to receive(:new).and_return(notify_service)
+    end
+
+    def make_request(body = {})
+      post :create, params: project_params, body: body.to_json, as: :json
+    end
+
+    context 'when notification service succeeds' do
+      let(:payload) do
+        {
+          title: 'Alert title',
+          hosts: 'https://gitlab.com'
+        }
+      end
+
+      let(:permitted_params) { ActionController::Parameters.new(payload).permit! }
+
+      it 'responds with ok' do
+        make_request
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+
+      it 'does not pass excluded parameters to the notify service' do
+        make_request(payload)
+
+        expect(Projects::Alerting::NotifyService)
+          .to have_received(:new)
+          .with(project, nil, permitted_params)
+      end
+    end
+
+    context 'when notification service fails' do
+      let(:service_response) { ServiceResponse.error(message: 'Unauthorized', http_status: 401) }
+
+      it 'responds with the service response' do
+        make_request
+
+        expect(response).to have_gitlab_http_status(:unauthorized)
+      end
+    end
+
+    context 'bearer token' do
+      context 'when set' do
+        it 'extracts bearer token' do
+          request.headers['HTTP_AUTHORIZATION'] = 'Bearer some token'
+
+          expect(notify_service).to receive(:execute).with('some token')
+
+          make_request
+        end
+
+        it 'pass nil if cannot extract a non-bearer token' do
+          request.headers['HTTP_AUTHORIZATION'] = 'some token'
+
+          expect(notify_service).to receive(:execute).with(nil)
+
+          make_request
+        end
+      end
+
+      context 'when missing' do
+        it 'passes nil' do
+          expect(notify_service).to receive(:execute).with(nil)
+
+          make_request
+        end
+      end
+    end
+  end
+
+  def project_params(opts = {})
+    opts.reverse_merge(namespace_id: project.namespace, project_id: project)
+  end
+end

spec/factories/services.rb

@@ -44,6 +44,16 @@ FactoryBot.define do
     end
   end
+  factory :alerts_service do
+    project
+    type { 'AlertsService' }
+    active { true }
+
+    trait :inactive do
+      active { false }
+    end
+  end
+
   factory :drone_ci_service do
     project
     active { true }

spec/features/projects/graph_spec.rb

@@ -22,20 +22,12 @@ describe 'Project Graph', :js do
     end
   end
-  shared_examples 'page should have languages graphs' do
-    it 'renders languages' do
-      expect(page).to have_content(/Ruby 66.* %/)
-      expect(page).to have_content(/JavaScript 22.* %/)
-    end
-  end
-
   context 'commits graph' do
     before do
       visit commits_project_graph_path(project, 'master')
     end
     it_behaves_like 'page should have commits graphs'
-    it_behaves_like 'page should have languages graphs'
   end
   context 'languages graph' do
@@ -44,7 +36,6 @@ describe 'Project Graph', :js do
     end
     it_behaves_like 'page should have commits graphs'
-    it_behaves_like 'page should have languages graphs'
   end
   context 'charts graph' do
@@ -53,7 +44,6 @@ describe 'Project Graph', :js do
     end
     it_behaves_like 'page should have commits graphs'
-    it_behaves_like 'page should have languages graphs'
   end
   context 'chart graph with HTML escaped branch name' do

spec/frontend/blob/components/__snapshots__/blob_header_spec.js.snap

@@ -12,13 +12,12 @@ exports[`Blob Header Default Actions rendering matches the snapshot 1`] = `
     class="file-actions d-none d-sm-block"
   >
     <viewer-switcher-stub
-      activeviewer="rich"
-      blob="[object Object]"
+      value="simple"
     />
       
     <default-actions-stub
-      activeviewer="rich"
-      blob="[object Object]"
+      activeviewer="simple"
+      rawpath="/flightjs/flight/snippets/51/raw"
     />
   </div>
 </div>