Add latest changes from gitlab-org/gitlab@master

app/workers/all_queues.yml

@@ -528,6 +528,55 @@
   :resource_boundary: :unknown
   :weight: 2
   :idempotent: 
+- :name: jira_importer:jira_import_advance_stage
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
+- :name: jira_importer:jira_import_stage_finish_import
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
+- :name: jira_importer:jira_import_stage_import_attachments
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
+- :name: jira_importer:jira_import_stage_import_issues
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
+- :name: jira_importer:jira_import_stage_import_labels
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
+- :name: jira_importer:jira_import_stage_import_notes
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
+- :name: jira_importer:jira_import_stage_start_import
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
 - :name: mail_scheduler:mail_scheduler_issue_due
   :feature_category: :issue_tracking
   :has_external_dependencies: 

app/workers/concerns/gitlab/jira_import/import_worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    module ImportWorker
+      extend ActiveSupport::Concern
+
+      included do
+        include ApplicationWorker
+        include Gitlab::JiraImport::QueueOptions
+      end
+
+      def perform(project_id)
+        project = Project.find_by(id: project_id) # rubocop: disable CodeReuse/ActiveRecord
+
+        return unless can_import?(project)
+
+        import(project)
+      end
+
+      private
+
+      def import(project)
+        raise NotImplementedError
+      end
+
+      def can_import?(project)
+        return false unless project
+        return false if Feature.disabled?(:jira_issue_import, project)
+
+        project.import_state.started?
+      end
+    end
+  end
+end

app/workers/concerns/gitlab/jira_import/queue_options.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    module QueueOptions
+      extend ActiveSupport::Concern
+
+      included do
+        queue_namespace :jira_importer
+        feature_category :importers
+
+        sidekiq_options retry: 5
+      end
+    end
+  end
+end

app/workers/gitlab/github_import/advance_stage_worker.rb

@@ -13,8 +13,6 @@ module Gitlab
       sidekiq_options dead: false
       feature_category :importers
-      private
-
       # The known importer stages and their corresponding Sidekiq workers.
       STAGES = {
         issues_and_diff_notes: Stage::ImportIssuesAndDiffNotesWorker,
@@ -23,6 +21,8 @@ module Gitlab
         finish: Stage::FinishImportWorker
       }.freeze
+      private
+
       def next_stage_worker(next_stage)
         STAGES.fetch(next_stage.to_sym)
       end

app/workers/gitlab/jira_import/advance_stage_worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
+      include ApplicationWorker
+      include QueueOptions
+      include ::Gitlab::Import::AdvanceStage
+
+      # The known importer stages and their corresponding Sidekiq workers.
+      STAGES = {
+        labels: Gitlab::JiraImport::Stage::ImportLabelsWorker,
+        issues: Gitlab::JiraImport::Stage::ImportIssuesWorker,
+        attachments: Gitlab::JiraImport::Stage::ImportAttachmentsWorker,
+        notes: Gitlab::JiraImport::Stage::ImportNotesWorker,
+        finish: Gitlab::JiraImport::Stage::FinishImportWorker
+      }.freeze
+
+      private
+
+      def next_stage_worker(next_stage)
+        STAGES.fetch(next_stage.to_sym)
+      end
+    end
+  end
+end

app/workers/gitlab/jira_import/stage/finish_import_worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    module Stage
+      class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker
+        include Gitlab::JiraImport::ImportWorker
+
+        private
+
+        def import(project)
+          project.after_import
+        ensure
+          project.import_data.becomes(JiraImportData).finish_import!
+          project.import_data.save!
+        end
+      end
+    end
+  end
+end

app/workers/gitlab/jira_import/stage/import_attachments_worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    module Stage
+      class ImportAttachmentsWorker # rubocop:disable Scalability/IdempotentWorker
+        include Gitlab::JiraImport::ImportWorker
+
+        private
+
+        def import(project)
+          # fake a attahcments import workers for now.
+          # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage
+          fake_waiter = JobWaiter.new
+
+          project.import_state.refresh_jid_expiration
+          Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { fake_waiter.key => fake_waiter.jobs_remaining }, :notes)
+        end
+      end
+    end
+  end
+end

app/workers/gitlab/jira_import/stage/import_issues_worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    module Stage
+      class ImportIssuesWorker # rubocop:disable Scalability/IdempotentWorker
+        include Gitlab::JiraImport::ImportWorker
+
+        private
+
+        def import(project)
+          # fake issues import workers for now
+          # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage
+          jobs_waiter = JobWaiter.new
+          project.import_state.refresh_jid_expiration
+
+          Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { jobs_waiter.key => jobs_waiter.jobs_remaining }, :attachments)
+        end
+      end
+    end
+  end
+end

app/workers/gitlab/jira_import/stage/import_labels_worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    module Stage
+      class ImportLabelsWorker # rubocop:disable Scalability/IdempotentWorker
+        include Gitlab::JiraImport::ImportWorker
+
+        private
+
+        def import(project)
+          # fake labels import workers for now
+          # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage
+          fake_waiter = JobWaiter.new
+          Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { fake_waiter.key => fake_waiter.jobs_remaining }, :issues)
+        end
+      end
+    end
+  end
+end

app/workers/gitlab/jira_import/stage/import_notes_worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    module Stage
+      class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker
+        include Gitlab::JiraImport::ImportWorker
+
+        private
+
+        def import(project)
+          # fake notes import workers for now
+          # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage
+          jobs_waiter = JobWaiter.new
+          project.import_state.refresh_jid_expiration
+
+          Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { jobs_waiter.key => jobs_waiter.jobs_remaining }, :finish)
+        end
+      end
+    end
+  end
+end

app/workers/gitlab/jira_import/stage/start_import_worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    module Stage
+      class StartImportWorker # rubocop:disable Scalability/IdempotentWorker
+        include ApplicationWorker
+        include ProjectStartImport
+        include ProjectImportOptions
+        include Gitlab::JiraImport::QueueOptions
+
+        attr_reader :project
+
+        def perform(project_id)
+          @project = Project.find_by(id: project_id) # rubocop: disable CodeReuse/ActiveRecord
+
+          return unless start_import
+
+          Gitlab::Import::SetAsyncJid.set_jid(project)
+
+          Gitlab::JiraImport::Stage::ImportLabelsWorker.perform_async(project.id)
+        end
+
+        private
+
+        def start_import
+          return false unless project
+          return false if Feature.disabled?(:jira_issue_import, project)
+          return true if start(project.import_state)
+
+          Gitlab::Import::Logger.info(
+            {
+              project_id: project.id,
+              project_path: project.full_path,
+              state: project&.import_status,
+              message: 'inconsistent state while importing'
+            }
+          )
+          false
+        end
+      end
+    end
+  end
+end

changelogs/unreleased/208894-fix-showing-only-free-namespaces-when-applying-licenses.yml

+---
+title: Fix managed_free_namespaces scope to only groups without a license or a free license
+merge_request: 27356
+author:
+type: fixed

changelogs/unreleased/208923-enable-batch-counting-for-some-individual-queries-5.yml

+---
+title: Optimize projects_mirrored_with_pipelines_enabled query performance in usage data
+merge_request: 27110
+author:
+type: performance

changelogs/unreleased/fix-invalid-milestones-when-moving-projects.yml

+---
+title: Fix invalid ancestor group milestones when moving projects
+merge_request: 27262
+author:
+type: fixed

config/gitlab.yml.example

@@ -459,6 +459,11 @@ production: &base
     elastic_index_bulk_cron_worker:
       cron: "*/1 * * * *"
+    # Elasticsearch metrics
+    # NOTE: This will only take effect if Elasticsearch is enabled.
+    elastic_metrics_update_worker:
+      cron: "*/1 * * * *"
+
   registry:
     # enabled: true
     # host: registry.example.com

config/initializers/1_settings.rb

@@ -546,6 +546,9 @@ Gitlab.ee do
   Settings.cron_jobs['elastic_index_bulk_cron_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['elastic_index_bulk_cron_worker']['cron'] ||= '*/1 * * * *'
   Settings.cron_jobs['elastic_index_bulk_cron_worker']['job_class'] ||= 'ElasticIndexBulkCronWorker'
+  Settings.cron_jobs['elastic_metrics_update_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['elastic_metrics_update_worker']['cron'] ||= '*/1 * * * *'
+  Settings.cron_jobs['elastic_metrics_update_worker']['job_class'] ||= 'ElasticMetricsUpdateWorker'
   Settings.cron_jobs['sync_seat_link_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['sync_seat_link_worker']['cron'] ||= "#{rand(60)} 0 * * *"
   Settings.cron_jobs['sync_seat_link_worker']['job_class'] = 'SyncSeatLinkWorker'

config/sidekiq_queues.yml

@@ -128,6 +128,8 @@
   - 1
 - - jira_connect
   - 1
+- - jira_importer
+  - 1
 - - ldap_group_sync
   - 2
 - - mail_scheduler

db/migrate/20200311141943_insert_ci_pipeline_schedules_plan_limits.rb

 # frozen_string_literal: true
 class InsertCiPipelineSchedulesPlanLimits < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
   DOWNTIME = false
   def change

db/migrate/20200312160532_add_index_on_mirror_and_id_to_projects.rb

+# frozen_string_literal: true
+
+class AddIndexOnMirrorAndIdToProjects < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  OLD_INDEX_NAME = 'index_projects_on_mirror_and_mirror_trigger_builds_both_true'
+  NEW_INDEX_NAME = 'index_projects_on_mirror_id_where_mirror_and_trigger_builds'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, :id, where: 'mirror = TRUE AND mirror_trigger_builds = TRUE', name: NEW_INDEX_NAME
+    remove_concurrent_index_by_name :projects, OLD_INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :projects, :id, where: 'mirror IS TRUE AND mirror_trigger_builds IS TRUE', name: OLD_INDEX_NAME
+    remove_concurrent_index_by_name :projects, NEW_INDEX_NAME
+  end
+end

db/schema.rb

@@ -3507,7 +3507,7 @@ ActiveRecord::Schema.define(version: 2020_03_13_123934) do
     t.index ["id"], name: "index_on_id_partial_with_legacy_storage", where: "((storage_version < 2) OR (storage_version IS NULL))"
     t.index ["id"], name: "index_projects_on_id_partial_for_visibility", unique: true, where: "(visibility_level = ANY (ARRAY[10, 20]))"
     t.index ["id"], name: "index_projects_on_id_service_desk_enabled", where: "(service_desk_enabled = true)"
-    t.index ["id"], name: "index_projects_on_mirror_and_mirror_trigger_builds_both_true", where: "((mirror IS TRUE) AND (mirror_trigger_builds IS TRUE))"
+    t.index ["id"], name: "index_projects_on_mirror_id_where_mirror_and_trigger_builds", where: "((mirror = true) AND (mirror_trigger_builds = true))"
     t.index ["last_activity_at", "id"], name: "index_projects_api_last_activity_at_id_desc", order: { id: :desc }
     t.index ["last_activity_at", "id"], name: "index_projects_api_vis20_last_activity_at", where: "(visibility_level = 20)"
     t.index ["last_activity_at", "id"], name: "index_projects_on_last_activity_at_and_id"