WIP

changelogs/unreleased/zj-cronjob-requeue-pending-delete.yml

 ---
-title: Requeue projects pending deletion
+title: Delete soft deleted entities from the database
 merge_request: 7067
 author: 

db/post_migrate/20161107200810_requeue_pending_delete.rb

-class RequeuePendingDelete < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-
-  # When using the methods "add_concurrent_index" or "add_column_with_default"
-  # you must disable the use of transactions as these methods can not run in an
-  # existing transaction. When using "add_concurrent_index" make sure that this
-  # method is the _only_ method called in the migration, any other changes
-  # should go in a separate migration. This ensures that upon failure _only_ the
-  # index creation fails and can be retried or reverted easily.
-  #
-  # To disable transactions uncomment the following line and remove these
-  # comments:
-  # disable_ddl_transaction!
-
-  def up
-    admin = User.find_by(admin: true)
-
-    Group.with_deleted.where.not(deleted_at: nil).find_each do |group|
-      GroupDestroyWorker.perform_async(group.id, admin.id)
-    end
-
-    Project.unscoped.where(pending_delete: true).find_each do |project|
-      ProjectDestroyWorker.perform_async(project.id, admin.id)
-    end
-  end
-
-  def down
-    # Noop
-  end
-end

db/post_migrate/20161121090906_delete_soft_deleted_entities.rb

+class DeleteSoftDeletedEntities < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  BATCH_SIZE = 128
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    [namespaces_sql, projects_sql].each do |sql|
+      loop do
+        deleted = connection.exec_query(sql)
+
+        break if deleted.rows.count.zero?
+
+        sleep(100)
+      end
+    end
+  end
+
+  def down
+    # Noop
+  end
+
+  def namespaces_sql
+    namespaces = Arel::Table.new(:namespaces)
+    Arel::DeleteManager.new(ActiveRecord::Base).  
+      from(namespaces).
+      where(namespaces[:id].in(
+        namespaces.project(namespaces[:id]).
+        where(namespaces[:deleted_at].not_eq(nil)).
+        take(BATCH_SIZE))
+      ).
+      to_sql
+  end
+
+  def projects_sql
+    projects = Arel::Table.new(:projects)
+    Arel::DeleteManager.new(ActiveRecord::Base).
+      from(projects).
+      where(projects[:id].in(
+        projects.project(projects[:id]).
+        where(projects[:pending_delete].eq(true)).
+        take(BATCH_SIZE))
+      ).
+      to_sql
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20161118183841) do
+ActiveRecord::Schema.define(version: 20161121090906) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"