Merge branch 'lfs_object_removal' into 'master'

Remove unreferenced LFS objects from DB and fs

Fixes #3666

See merge request !5901

Former-commit-id: 1167c0be

CHANGELOG.md

@@ -29,6 +29,7 @@ Please view this file on the master branch, on stable branches it's out of date.
   - New issue board list dropdown stays open after adding a new list
   - Fix: Backup restore doesn't clear cache
   - API: Fix project deploy keys 400 and 500 errors when adding an existing key. !6784 (Joshua Welsh)
+  - Add job for removal of unreferenced LFS objects from both the database and the filesystem (Frank Groeneveld)
   - Replace jquery.cookie plugin with js.cookie !7085
   - Use MergeRequestsClosingIssues cache data on Issue#closed_by_merge_requests method
   - Fix Sign in page 'Forgot your password?' link overlaps on medium-large screens

app/models/lfs_object.rb

@@ -17,4 +17,10 @@ class LfsObject < ActiveRecord::Base
   def project_allowed_access?(project)
     projects.exists?(storage_project(project).id)
   end
+
+  def self.destroy_unreferenced
+    joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id")
+        .where(lfs_objects_projects: { id: nil })
+        .destroy_all
+  end
 end

app/workers/remove_unreferenced_lfs_objects_worker.rb

+class RemoveUnreferencedLfsObjectsWorker
+  include Sidekiq::Worker
+  include CronjobQueue
+
+  def perform
+    LfsObject.destroy_unreferenced
+  end
+end

config/initializers/1_settings.rb

@@ -307,6 +307,9 @@ Settings.cron_jobs['prune_old_events_worker']['job_class'] = 'PruneOldEventsWork
 Settings.cron_jobs['trending_projects_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['trending_projects_worker']['cron'] = '0 1 * * *'
 Settings.cron_jobs['trending_projects_worker']['job_class'] = 'TrendingProjectsWorker'
+Settings.cron_jobs['remove_unreferenced_lfs_objects_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['cron'] ||= '20 0 * * *'
+Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'RemoveUnreferencedLfsObjectsWorker'
 #
 # GitLab Shell

spec/workers/remove_unreferenced_lfs_objects_worker_spec.rb

+require 'spec_helper'
+
+describe RemoveUnreferencedLfsObjectsWorker do
+  let(:worker) { RemoveUnreferencedLfsObjectsWorker.new }
+
+  describe '#perform' do
+    let!(:unreferenced_lfs_object1) { create(:lfs_object, oid: '1') }
+    let!(:unreferenced_lfs_object2) { create(:lfs_object, oid: '2') }
+    let!(:project1) { create(:empty_project, lfs_enabled: true) }
+    let!(:project2) { create(:empty_project, lfs_enabled: true) }
+    let!(:referenced_lfs_object1) { create(:lfs_object, oid: '3') }
+    let!(:referenced_lfs_object2) { create(:lfs_object, oid: '4') }
+    let!(:lfs_objects_project1_1) do
+      create(:lfs_objects_project,
+                project: project1,
+                lfs_object: referenced_lfs_object1
+            )
+    end
+    let!(:lfs_objects_project2_1) do
+      create(:lfs_objects_project,
+                project: project2,
+                lfs_object: referenced_lfs_object1
+            )
+    end
+    let!(:lfs_objects_project1_2) do
+      create(:lfs_objects_project,
+                project: project1,
+                lfs_object: referenced_lfs_object2
+            )
+    end
+
+    it 'removes unreferenced lfs objects' do
+      worker.perform
+
+      expect(LfsObject.where(id: unreferenced_lfs_object1.id)).to be_empty
+      expect(LfsObject.where(id: unreferenced_lfs_object2.id)).to be_empty
+    end
+
+    it 'leaves referenced lfs objects' do
+      worker.perform
+
+      expect(referenced_lfs_object1.reload).to be_present
+      expect(referenced_lfs_object2.reload).to be_present
+    end
+
+    it 'removes unreferenced lfs objects after project removal' do
+      project1.destroy
+
+      worker.perform
+
+      expect(referenced_lfs_object1.reload).to be_present
+      expect(LfsObject.where(id: referenced_lfs_object2.id)).to be_empty
+    end
+  end
+end