Merge branch 'drop-old-builds' into 'master'

Cleanup stuck CI builds

Fixes #3143

/cc @dzaporozhets @jacobvosmaer 

See merge request !1655

CHANGELOG

@@ -37,6 +37,7 @@ v 8.1.0
   - Fix duplicate repositories in GitHub import page (Stan Hu)
   - Redirect to a default path if HTTP_REFERER is not set (Stan Hu)
   - Adds ability to create directories using the web editor (Ben Ford)
+  - Cleanup stuck CI builds
 v 8.1.0 (unreleased)
   - Send an email to admin email when a user is reported for spam (Jonathan Rochkind)

app/workers/stuck_ci_builds_worker.rb

+class StuckCiBuildsWorker
+  include Sidekiq::Worker
+  include Sidetiq::Schedulable
+
+  BUILD_STUCK_TIMEOUT = 1.day
+
+  recurrence { daily }
+
+  def perform
+    Rails.logger.info 'Cleaning stuck builds'
+
+    builds = Ci::Build.running_or_pending.where('updated_at < ?', BUILD_STUCK_TIMEOUT.ago)
+    builds.find_each(batch_size: 50).each do |build|
+      Rails.logger.debug "Dropping stuck #{build.status} build #{build.id} for runner #{build.runner_id}"
+      build.drop
+    end
+  end
+end

spec/workers/stuck_ci_builds_worker_spec.rb

+require "spec_helper"
+
+describe StuckCiBuildsWorker do
+  let!(:build) { create :ci_build }
+
+  subject do
+    build.reload
+    build.status
+  end
+
+  %w(pending running).each do |status|
+    context "#{status} build" do
+      before do
+        build.update!(status: status)
+      end
+
+      it 'gets dropped if it was updated over 2 days ago' do
+        build.update!(updated_at: 2.day.ago)
+        StuckCiBuildsWorker.new.perform
+        is_expected.to eq('failed')
+      end
+
+      it "is still #{status}" do
+        build.update!(updated_at: 1.minute.ago)
+        StuckCiBuildsWorker.new.perform
+        is_expected.to eq(status)
+      end
+    end
+  end
+
+  %w(success failed canceled).each do |status|
+    context "#{status} build" do
+      before do
+        build.update!(status: status)
+      end
+
+      it "is still #{status}" do
+        build.update!(updated_at: 2.day.ago)
+        StuckCiBuildsWorker.new.perform
+        is_expected.to eq(status)
+      end
+    end
+  end
+end