Skip to content
Snippets Groups Projects
Commit cf9c831e authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre
Browse files

Merge branch '426643-geo-remove-legacy-sidekiq-workers' into 'master'

parents 60343af2 ef58c101
No related branches found
No related tags found
No related merge requests found
Showing
with 25 additions and 373 deletions
Loading
Loading
@@ -1028,7 +1028,6 @@ Gitlab/NamespacedClass:
- 'ee/app/workers/elastic_namespace_rollout_worker.rb'
- 'ee/app/workers/elastic_remove_expired_namespace_subscriptions_from_index_cron_worker.rb'
- 'ee/app/workers/elastic_wiki_indexer_worker.rb'
- 'ee/app/workers/geo_repository_destroy_worker.rb'
- 'ee/app/workers/group_saml_group_sync_worker.rb'
- 'ee/app/workers/historical_data_worker.rb'
- 'ee/app/workers/import_software_licenses_worker.rb'
Loading
Loading
Loading
Loading
@@ -375,7 +375,6 @@ Gitlab/StrongMemoizeAttr:
- 'ee/app/services/vulnerability_feedback/create_service.rb'
- 'ee/app/services/vulnerability_feedback/destroy_service.rb'
- 'ee/app/workers/auth/saml_group_sync_worker.rb'
- 'ee/app/workers/geo/repository_cleanup_worker.rb'
- 'ee/app/workers/geo/scheduler/scheduler_worker.rb'
- 'ee/app/workers/status_page/publish_worker.rb'
- 'ee/lib/api/analytics/project_deployment_frequency.rb'
Loading
Loading
Loading
Loading
@@ -1083,16 +1083,10 @@ Layout/LineLength:
- 'ee/app/workers/elastic_commit_indexer_worker.rb'
- 'ee/app/workers/elastic_delete_project_worker.rb'
- 'ee/app/workers/elastic_namespace_rollout_worker.rb'
- 'ee/app/workers/geo/batch/project_registry_scheduler_worker.rb'
- 'ee/app/workers/geo/batch/project_registry_worker.rb'
- 'ee/app/workers/geo/destroy_worker.rb'
- 'ee/app/workers/geo/repositories_clean_up_worker.rb'
- 'ee/app/workers/geo/repository_shard_sync_worker.rb'
- 'ee/app/workers/geo/repository_verification/secondary/shard_worker.rb'
- 'ee/app/workers/geo/scheduler/scheduler_worker.rb'
- 'ee/app/workers/geo/secondary/registry_consistency_worker.rb'
- 'ee/app/workers/geo/verification_worker.rb'
- 'ee/app/workers/geo_repository_destroy_worker.rb'
- 'ee/app/workers/repository_update_mirror_worker.rb'
- 'ee/app/workers/security/orchestration_policy_rule_schedule_namespace_worker.rb'
- 'ee/app/workers/security/orchestration_policy_rule_schedule_worker.rb'
Loading
Loading
Loading
Loading
@@ -114,8 +114,6 @@ Lint/RedundantCopDisableDirective:
- 'ee/app/services/ee/search_service.rb'
- 'ee/app/services/security/token_revocation_service.rb'
- 'ee/app/workers/ee/issuable_export_csv_worker.rb'
- 'ee/app/workers/geo/repository_shard_sync_worker.rb'
- 'ee/app/workers/geo/repository_verification/secondary/shard_worker.rb'
- 'ee/app/workers/scan_security_report_secrets_worker.rb'
- 'ee/app/workers/security/orchestration_policy_rule_schedule_worker.rb'
- 'ee/db/geo/migrate/20210504143244_add_verification_to_merge_request_diff_registry.rb'
Loading
Loading
Loading
Loading
@@ -21,8 +21,6 @@ Rails/Pluck:
- 'ee/app/models/boards/epic_list.rb'
- 'ee/app/models/concerns/geo/verification_state.rb'
- 'ee/app/services/concerns/incident_management/oncall_rotations/shared_rotation_logic.rb'
- 'ee/app/workers/geo/repository_shard_sync_worker.rb'
- 'ee/app/workers/geo/repository_verification/secondary/shard_worker.rb'
- 'ee/app/workers/geo/scheduler/scheduler_worker.rb'
- 'ee/lib/banzai/filter/references/iteration_reference_filter.rb'
- 'ee/lib/ee/gitlab/auth/ldap/person.rb'
Loading
Loading
Loading
Loading
@@ -344,8 +344,6 @@ SidekiqLoadBalancing/WorkerDataConsistency:
- 'ee/app/workers/epics/new_epic_issue_worker.rb'
- 'ee/app/workers/epics/update_cached_metadata_worker.rb'
- 'ee/app/workers/epics/update_epics_dates_worker.rb'
- 'ee/app/workers/geo/batch/project_registry_scheduler_worker.rb'
- 'ee/app/workers/geo/batch/project_registry_worker.rb'
- 'ee/app/workers/geo/batch_event_create_worker.rb'
- 'ee/app/workers/geo/container_repository_sync_worker.rb'
- 'ee/app/workers/geo/create_repository_updated_event_worker.rb'
Loading
Loading
@@ -353,17 +351,9 @@ SidekiqLoadBalancing/WorkerDataConsistency:
- 'ee/app/workers/geo/event_worker.rb'
- 'ee/app/workers/geo/file_removal_worker.rb'
- 'ee/app/workers/geo/hashed_storage_attachments_migration_worker.rb'
- 'ee/app/workers/geo/hashed_storage_migration_worker.rb'
- 'ee/app/workers/geo/metrics_update_worker.rb'
- 'ee/app/workers/geo/project_sync_worker.rb'
- 'ee/app/workers/geo/prune_event_log_worker.rb'
- 'ee/app/workers/geo/rename_repository_worker.rb'
- 'ee/app/workers/geo/repositories_clean_up_worker.rb'
- 'ee/app/workers/geo/repository_cleanup_worker.rb'
- 'ee/app/workers/geo/repository_verification/primary/single_worker.rb'
- 'ee/app/workers/geo/repository_verification/secondary/single_worker.rb'
- 'ee/app/workers/geo/reverification_batch_worker.rb'
- 'ee/app/workers/geo/scheduler/per_shard_scheduler_worker.rb'
- 'ee/app/workers/geo/scheduler/scheduler_worker.rb'
- 'ee/app/workers/geo/secondary/registry_consistency_worker.rb'
- 'ee/app/workers/geo/secondary_usage_data_cron_worker.rb'
Loading
Loading
@@ -374,7 +364,6 @@ SidekiqLoadBalancing/WorkerDataConsistency:
- 'ee/app/workers/geo/verification_state_backfill_worker.rb'
- 'ee/app/workers/geo/verification_timeout_worker.rb'
- 'ee/app/workers/geo/verification_worker.rb'
- 'ee/app/workers/geo_repository_destroy_worker.rb'
- 'ee/app/workers/gitlab_subscriptions/schedule_refresh_seats_worker.rb'
- 'ee/app/workers/gitlab_subscriptions/trials/apply_trial_worker.rb'
- 'ee/app/workers/group_saml_group_sync_worker.rb'
Loading
Loading
Loading
Loading
@@ -1696,33 +1696,15 @@ Style/InlineDisableAnnotation:
- 'ee/app/workers/emails/abandoned_trial_emails_cron_worker.rb'
- 'ee/app/workers/epics/new_epic_issue_worker.rb'
- 'ee/app/workers/epics/update_epics_dates_worker.rb'
- 'ee/app/workers/geo/batch/project_registry_scheduler_worker.rb'
- 'ee/app/workers/geo/batch/project_registry_worker.rb'
- 'ee/app/workers/geo/bulk_mark_pending_batch_worker.rb'
- 'ee/app/workers/geo/bulk_mark_verification_pending_batch_worker.rb'
- 'ee/app/workers/geo/container_repository_sync_worker.rb'
- 'ee/app/workers/geo/event_worker.rb'
- 'ee/app/workers/geo/file_removal_worker.rb'
- 'ee/app/workers/geo/hashed_storage_attachments_migration_worker.rb'
- 'ee/app/workers/geo/hashed_storage_migration_worker.rb'
- 'ee/app/workers/geo/metrics_update_worker.rb'
- 'ee/app/workers/geo/prune_event_log_worker.rb'
- 'ee/app/workers/geo/rename_repository_worker.rb'
- 'ee/app/workers/geo/repositories_clean_up_worker.rb'
- 'ee/app/workers/geo/repository_cleanup_worker.rb'
- 'ee/app/workers/geo/repository_shard_sync_worker.rb'
- 'ee/app/workers/geo/repository_sync_worker.rb'
- 'ee/app/workers/geo/repository_verification/primary/batch_worker.rb'
- 'ee/app/workers/geo/repository_verification/primary/shard_worker.rb'
- 'ee/app/workers/geo/repository_verification/primary/single_worker.rb'
- 'ee/app/workers/geo/repository_verification/secondary/scheduler_worker.rb'
- 'ee/app/workers/geo/repository_verification/secondary/shard_worker.rb'
- 'ee/app/workers/geo/repository_verification/secondary/single_worker.rb'
- 'ee/app/workers/geo/scheduler/per_shard_scheduler_worker.rb'
- 'ee/app/workers/geo/scheduler/primary/per_shard_scheduler_worker.rb'
- 'ee/app/workers/geo/scheduler/primary/scheduler_worker.rb'
- 'ee/app/workers/geo/scheduler/scheduler_worker.rb'
- 'ee/app/workers/geo/scheduler/secondary/per_shard_scheduler_worker.rb'
- 'ee/app/workers/geo/scheduler/secondary/scheduler_worker.rb'
- 'ee/app/workers/geo/secondary/registry_consistency_worker.rb'
- 'ee/app/workers/geo/secondary_usage_data_cron_worker.rb'
Loading
Loading
@@ -1730,7 +1712,6 @@ Style/InlineDisableAnnotation:
- 'ee/app/workers/geo/sync_timeout_cron_worker.rb'
- 'ee/app/workers/geo/verification_cron_worker.rb'
- 'ee/app/workers/geo/verification_state_backfill_worker.rb'
- 'ee/app/workers/geo_repository_destroy_worker.rb'
- 'ee/app/workers/gitlab_subscriptions/add_on_purchases/schedule_bulk_refresh_user_assignments_worker.rb'
- 'ee/app/workers/gitlab_subscriptions/notify_seats_exceeded_batch_worker.rb'
- 'ee/app/workers/gitlab_subscriptions/refresh_seats_worker.rb'
Loading
Loading
Loading
Loading
@@ -80,7 +80,6 @@ Style/SymbolProc:
- 'ee/app/services/security/scanned_resources_counting_service.rb'
- 'ee/app/services/timebox_report_service.rb'
- 'ee/app/services/vulnerabilities/historical_statistics/deletion_service.rb'
- 'ee/app/workers/geo/batch/project_registry_worker.rb'
- 'ee/app/workers/geo/sync_timeout_cron_worker.rb'
- 'ee/app/workers/geo/verification_cron_worker.rb'
- 'ee/lib/api/entities/pending_member.rb'
Loading
Loading
# frozen_string_literal: true
class RemoveGeoPrimaryDeprecatedWorkersJobInstances < Gitlab::Database::Migration[2.2]
DEPRECATED_JOB_CLASSES = %w[
Geo::RepositoryVerification::Primary::BatchWorker
Geo::RepositoryVerification::Primary::ShardWorker
Geo::RepositoryVerification::Primary::SingleWorker
Geo::RepositoryVerification::Secondary::SingleWorker
Geo::Scheduler::Primary::PerShardSchedulerWorker
Geo::Scheduler::Primary::SchedulerWorker
]
disable_ddl_transaction!
milestone '16.7'
def up
sidekiq_remove_jobs(job_klasses: DEPRECATED_JOB_CLASSES)
end
def down
# This migration removes any instances of deprecated workers and cannot be undone.
end
end
f0f89526db19621991482cc7fce3d95d8762d020d06ecd3bf96c587cca71d015
\ No newline at end of file
Loading
Loading
@@ -237,69 +237,6 @@
:weight: 1
:idempotent: true
:tags: []
- :name: cronjob:geo_repository_sync
:worker_name: Geo::RepositorySyncWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: cronjob:geo_repository_verification_primary_batch
:worker_name: Geo::RepositoryVerification::Primary::BatchWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: cronjob:geo_repository_verification_secondary_scheduler
:worker_name: Geo::RepositoryVerification::Secondary::SchedulerWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: cronjob:geo_repository_verification_secondary_shard
:worker_name: Geo::RepositoryVerification::Secondary::ShardWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: cronjob:geo_scheduler_per_shard_scheduler
:worker_name: Geo::Scheduler::PerShardSchedulerWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: cronjob:geo_scheduler_primary_per_shard_scheduler
:worker_name: Geo::Scheduler::Primary::PerShardSchedulerWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: cronjob:geo_scheduler_secondary_per_shard_scheduler
:worker_name: Geo::Scheduler::Secondary::PerShardSchedulerWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: cronjob:geo_secondary_registry_consistency
:worker_name: Geo::Secondary::RegistryConsistencyWorker
:feature_category: :geo_replication
Loading
Loading
@@ -678,24 +615,6 @@
:weight: 1
:idempotent: true
:tags: []
- :name: geo:geo_batch_project_registry
:worker_name: Geo::Batch::ProjectRegistryWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_batch_project_registry_scheduler
:worker_name: Geo::Batch::ProjectRegistrySchedulerWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_bulk_mark_pending_batch
:worker_name: Geo::BulkMarkPendingBatchWorker
:feature_category: :geo_replication
Loading
Loading
@@ -768,96 +687,6 @@
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_hashed_storage_migration
:worker_name: Geo::HashedStorageMigrationWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_project_sync
:worker_name: Geo::ProjectSyncWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: geo:geo_rename_repository
:worker_name: Geo::RenameRepositoryWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_repositories_clean_up
:worker_name: Geo::RepositoriesCleanUpWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_repository_cleanup
:worker_name: Geo::RepositoryCleanupWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_repository_destroy
:worker_name: GeoRepositoryDestroyWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_repository_shard_sync
:worker_name: Geo::RepositoryShardSyncWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_repository_verification_primary_shard
:worker_name: Geo::RepositoryVerification::Primary::ShardWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_repository_verification_primary_single
:worker_name: Geo::RepositoryVerification::Primary::SingleWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_repository_verification_secondary_single
:worker_name: Geo::RepositoryVerification::Secondary::SingleWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_resync_direct_upload_job_artifact_registry
:worker_name: Geo::ResyncDirectUploadJobArtifactRegistryWorker
:feature_category: :geo_replication
Loading
Loading
@@ -876,15 +705,6 @@
:weight: 1
:idempotent: true
:tags: []
- :name: geo:geo_scheduler_primary_scheduler
:worker_name: Geo::Scheduler::Primary::SchedulerWorker
:feature_category: :geo_replication
:has_external_dependencies: false
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: geo:geo_scheduler_scheduler
:worker_name: Geo::Scheduler::SchedulerWorker
:feature_category: :geo_replication
Loading
Loading
# frozen_string_literal: true
module Geo
module Batch
# Responsible for scheduling multiple jobs to mark Project Registries as requiring syncing or verification.
#
# This class includes an Exclusive Lease guard and only one can be executed at the same time
# If multiple jobs are scheduled, only one will run and the others will drop forever.
class ProjectRegistrySchedulerWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
data_consistency :always
sidekiq_options retry: 3
include GeoQueue
loggable_arguments 0
def perform(operation); end
end
end
end
# frozen_string_literal: true
module Geo
module Batch
# Responsible for scheduling multiple jobs to mark Project Registries as requiring syncing or verification.
#
# This class includes an Exclusive Lease guard and only one can be executed at the same time
# If multiple jobs are scheduled, only one will run and the others will drop forever.
class ProjectRegistryWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
data_consistency :always
sidekiq_options retry: 3
include GeoQueue
loggable_arguments 0, 1
def perform(operation, range); end
end
end
end
# frozen_string_literal: true
module Geo
class HashedStorageMigrationWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
data_consistency :always
include GeoQueue
loggable_arguments 1, 2
def perform(project_id, old_disk_path, new_disk_path, old_storage_version); end
end
end
# frozen_string_literal: true
module Geo
class ProjectSyncWorker
include ApplicationWorker
include GeoQueue
# Do not enqueue another instance of this Worker with the same args
# if one is currently enqueued or executing. If deduplication occurs,
# then reschedule the job once after the first job finishes, to
# ensure all changes get replicated.
deduplicate :until_executed, if_deduplicated: :reschedule_once
idempotent!
data_consistency :always
sidekiq_options retry: 1, dead: false
sidekiq_retry_in { |count| 30 * count }
sidekiq_retries_exhausted do |msg, _|
Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
end
loggable_arguments 1
def perform(project_id, options = {}); end
end
end
# frozen_string_literal: true
module Geo
class RenameRepositoryWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
data_consistency :always
sidekiq_options retry: 3
include GeoQueue
loggable_arguments 1, 2
def perform(project_id, old_disk_path, new_disk_path); end
end
end
# frozen_string_literal: true
module Geo
class RepositoriesCleanUpWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
data_consistency :always
include GeoQueue
def perform(geo_node_id); end
end
end
# frozen_string_literal: true
module Geo
class RepositoryCleanupWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
data_consistency :always
sidekiq_options retry: 3
include GeoQueue
loggable_arguments 1, 2, 3
def perform(project_id, name, disk_path, storage_name); end
end
end
# frozen_string_literal: true
module Geo
class RepositoryShardSyncWorker < Geo::Scheduler::Secondary::SchedulerWorker # rubocop:disable Scalability/IdempotentWorker
sidekiq_options retry: false
loggable_arguments 0
def perform(shard_name); end
end
end
# frozen_string_literal: true
module Geo
class RepositorySyncWorker < Geo::Scheduler::Secondary::PerShardSchedulerWorker # rubocop:disable Scalability/IdempotentWorker
def schedule_job(shard_name); end
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment