Merge branch '31207-clean-locked-merge-requests' into 'master'

Resolve "Store MergeWorker JID on merge request, and clean up stuck merges"

Closes #31207

See merge request !13207

.flayignore

@@ -3,4 +3,5 @@ lib/gitlab/sanitizers/svg/whitelist.rb
 lib/gitlab/diff/position_tracer.rb
 app/policies/project_policy.rb
 app/models/concerns/relative_positioning.rb
+app/workers/stuck_merge_jobs_worker.rb
 lib/gitlab/redis/*.rb

app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_locked.js → app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_merging.js

 import statusIcon from '../mr_widget_status_icon';
 export default {
-  name: 'MRWidgetLocked',
+  name: 'MRWidgetMerging',
   props: {
     mr: { type: Object, required: true },
   },
@@ -13,7 +13,7 @@ export default {
       <status-icon status="loading" />
       <div class="media-body">
         <h4>
-          This merge request is in the process of being merged, during which time it is locked and cannot be closed
+          This merge request is in the process of being merged
         </h4>
         <section class="mr-info-list">
           <p>

app/assets/javascripts/vue_merge_request_widget/dependencies.js

@@ -19,7 +19,7 @@ export { default as WidgetRelatedLinks } from './components/mr_widget_related_li
 export { default as MergedState } from './components/states/mr_widget_merged';
 export { default as FailedToMerge } from './components/states/mr_widget_failed_to_merge';
 export { default as ClosedState } from './components/states/mr_widget_closed';
-export { default as LockedState } from './components/states/mr_widget_locked';
+export { default as MergingState } from './components/states/mr_widget_merging';
 export { default as WipState } from './components/states/mr_widget_wip';
 export { default as ArchivedState } from './components/states/mr_widget_archived';
 export { default as ConflictsState } from './components/states/mr_widget_conflicts';

app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js

@@ -8,7 +8,7 @@ import {
   WidgetRelatedLinks,
   MergedState,
   ClosedState,
-  LockedState,
+  MergingState,
   WipState,
   ArchivedState,
   ConflictsState,
@@ -212,7 +212,7 @@ export default {
     'mr-widget-related-links': WidgetRelatedLinks,
     'mr-widget-merged': MergedState,
     'mr-widget-closed': ClosedState,
-    'mr-widget-locked': LockedState,
+    'mr-widget-merging': MergingState,
     'mr-widget-failed-to-merge': FailedToMerge,
     'mr-widget-wip': WipState,
     'mr-widget-archived': ArchivedState,

app/assets/javascripts/vue_merge_request_widget/stores/mr_widget_store.js

@@ -73,6 +73,7 @@ export default class MergeRequestStore {
     this.canCancelAutomaticMerge = !!data.cancel_merge_when_pipeline_succeeds_path;
     this.hasSHAChanged = this.sha !== data.diff_head_sha;
     this.canBeMerged = data.can_be_merged || false;
+    this.mergeOngoing = data.merge_ongoing;
     // Cherry-pick and Revert actions related
     this.canCherryPickInCurrentMR = currentUser.can_cherry_pick_on_current_merge_request || false;
@@ -94,6 +95,11 @@ export default class MergeRequestStore {
   }
   setState(data) {
+    if (this.mergeOngoing) {
+      this.state = 'merging';
+      return;
+    }
+
     if (this.isOpen) {
       this.state = getStateKey.call(this, data);
     } else {
@@ -104,9 +110,6 @@ export default class MergeRequestStore {
         case 'closed':
           this.state = 'closed';
           break;
-        case 'locked':
-          this.state = 'locked';
-          break;
         default:
           this.state = null;
       }

app/assets/javascripts/vue_merge_request_widget/stores/state_maps.js

 const stateToComponentMap = {
   merged: 'mr-widget-merged',
   closed: 'mr-widget-closed',
-  locked: 'mr-widget-locked',
+  merging: 'mr-widget-merging',
   conflicts: 'mr-widget-conflicts',
   missingBranch: 'mr-widget-missing-branch',
   workInProgress: 'mr-widget-wip',
@@ -20,7 +20,7 @@ const stateToComponentMap = {
 };
 const statesToShowHelpWidget = [
-  'locked',
+  'merging',
   'conflicts',
   'workInProgress',
   'readyToMerge',

app/controllers/projects/merge_requests_controller.rb

@@ -67,11 +67,6 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
         @noteable = @merge_request
         @commits_count = @merge_request.commits_count
-        if @merge_request.locked_long_ago?
-          @merge_request.unlock_mr
-          @merge_request.close
-        end
-
         labels
         set_pipeline_variables

app/models/merge_request.rb

@@ -8,6 +8,7 @@ class MergeRequest < ActiveRecord::Base
   include CreatedAtFilterable
   ignore_column :position
+  ignore_column :locked_at
   belongs_to :target_project, class_name: "Project"
   belongs_to :source_project, class_name: "Project"
@@ -61,16 +62,6 @@ class MergeRequest < ActiveRecord::Base
       transition locked: :opened
     end
-    after_transition any => :locked do |merge_request, transition|
-      merge_request.locked_at = Time.now
-      merge_request.save
-    end
-
-    after_transition locked: (any - :locked) do |merge_request, transition|
-      merge_request.locked_at = nil
-      merge_request.save
-    end
-
     state :opened
     state :closed
     state :merged
@@ -392,6 +383,12 @@ class MergeRequest < ActiveRecord::Base
                'Source project is not a fork of the target project'
   end
+  def merge_ongoing?
+    return false unless merge_jid
+
+    Gitlab::SidekiqStatus.num_running([merge_jid]) > 0
+  end
+
   def closed_without_fork?
     closed? && source_project_missing?
   end
@@ -725,12 +722,6 @@ class MergeRequest < ActiveRecord::Base
     end
   end
-  def locked_long_ago?
-    return false unless locked?
-
-    locked_at.nil? || locked_at < (Time.now - 1.day)
-  end
-
   def has_ci?
     has_ci_integration = source_project.try(:ci_service)
     uses_gitlab_ci = all_pipelines.any?

app/serializers/merge_request_entity.rb

@@ -2,7 +2,6 @@ class MergeRequestEntity < IssuableEntity
   include RequestAwareEntity
   expose :in_progress_merge_commit_sha
-  expose :locked_at
   expose :merge_commit_sha
   expose :merge_error
   expose :merge_params
@@ -32,6 +31,7 @@ class MergeRequestEntity < IssuableEntity
   expose :head_pipeline, with: PipelineDetailsEntity, as: :pipeline
   # Booleans
+  expose :merge_ongoing?, as: :merge_ongoing
   expose :work_in_progress?, as: :work_in_progress
   expose :source_branch_exists?, as: :source_branch_exists
   expose :mergeable_discussions_state?, as: :mergeable_discussions_state

app/workers/merge_worker.rb

@@ -7,6 +7,8 @@ class MergeWorker
     current_user = User.find(current_user_id)
     merge_request = MergeRequest.find(merge_request_id)
+    merge_request.update_column(:merge_jid, jid)
+
     MergeRequests::MergeService.new(merge_request.target_project, current_user, params)
       .execute(merge_request)
   end

app/workers/stuck_merge_jobs_worker.rb

+class StuckMergeJobsWorker
+  include Sidekiq::Worker
+  include CronjobQueue
+
+  def perform
+    stuck_merge_requests.find_in_batches(batch_size: 100) do |group|
+      jids = group.map(&:merge_jid)
+
+      # Find the jobs that aren't currently running or that exceeded the threshold.
+      completed_jids = Gitlab::SidekiqStatus.completed_jids(jids)
+
+      if completed_jids.any?
+        completed_ids = group.select { |merge_request| completed_jids.include?(merge_request.merge_jid) }.map(&:id)
+
+        apply_current_state!(completed_jids, completed_ids)
+      end
+    end
+  end
+
+  private
+
+  def apply_current_state!(completed_jids, completed_ids)
+    merge_requests = MergeRequest.where(id: completed_ids)
+
+    merge_requests.where.not(merge_commit_sha: nil).update_all(state: :merged)
+    merge_requests.where(merge_commit_sha: nil).update_all(state: :opened)
+
+    Rails.logger.info("Updated state of locked merge jobs. JIDs: #{completed_jids.join(', ')}")
+  end
+
+  def stuck_merge_requests
+    MergeRequest.select('id, merge_jid').with_state(:locked).where.not(merge_jid: nil).reorder(nil)
+  end
+end

changelogs/unreleased/31207-clean-locked-merge-requests.yml

+---
+title: Unlock stuck merge request and set the proper state
+merge_request: 13207
+author:

config/initializers/1_settings.rb

@@ -395,6 +395,10 @@ Settings.cron_jobs['remove_old_web_hook_logs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['remove_old_web_hook_logs_worker']['cron'] ||= '40 0 * * *'
 Settings.cron_jobs['remove_old_web_hook_logs_worker']['job_class'] = 'RemoveOldWebHookLogsWorker'
+Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
+Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
+
 #
 # GitLab Shell
 #

db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb

+class AddMergeJidToMergeRequests < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :merge_requests, :merge_jid, :string
+  end
+end

db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb

+class RemoveLockedAtColumnFromMergeRequests < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    remove_column :merge_requests, :locked_at
+  end
+
+  def down
+    add_column :merge_requests, :locked_at, :datetime_with_timezone
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20170803130232) do
+ActiveRecord::Schema.define(version: 20170807160457) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -850,7 +850,6 @@ ActiveRecord::Schema.define(version: 20170803130232) do
     t.integer "target_project_id", null: false
     t.integer "iid"
     t.text "description"
-    t.datetime "locked_at"
     t.integer "updated_by_id"
     t.text "merge_error"
     t.text "merge_params"
@@ -868,6 +867,7 @@ ActiveRecord::Schema.define(version: 20170803130232) do
     t.integer "last_edited_by_id"
     t.integer "head_pipeline_id"
     t.boolean "ref_fetched"
+    t.string "merge_jid"
   end
   add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree

doc/user/project/integrations/webhooks.md

@@ -438,7 +438,6 @@ X-Gitlab-Event: Note Hook
     "iid": 1,
     "description": "Et voluptas corrupti assumenda temporibus. Architecto cum animi eveniet amet asperiores. Vitae numquam voluptate est natus sit et ad id.",
     "position": 0,
-    "locked_at": null,
     "source":{
       "name":"Gitlab Test",
       "description":"Aut reprehenderit ut est.",

lib/gitlab/import_export/import_export.yml

@@ -101,6 +101,7 @@ excluded_attributes:
   merge_requests:
     - :milestone_id
     - :ref_fetched
+    - :merge_jid
   award_emoji:
     - :awardable_id
   statuses:

spec/features/merge_requests/widget_spec.rb

@@ -219,4 +219,17 @@ describe 'Merge request', :js do
       expect(page).to have_field('remove-source-branch-input', disabled: true)
     end
   end
+
+  context 'ongoing merge process' do
+    it 'shows Merging state' do
+      allow_any_instance_of(MergeRequest).to receive(:merge_ongoing?).and_return(true)
+
+      visit project_merge_request_path(project, merge_request)
+
+      wait_for_requests
+
+      expect(page).not_to have_button('Merge')
+      expect(page).to have_content('This merge request is in the process of being merged')
+    end
+  end
 end

spec/fixtures/api/schemas/entities/merge_request.json

@@ -19,7 +19,6 @@
     "human_time_estimate": { "type": ["integer", "null"] },
     "human_total_time_spent": { "type": ["integer", "null"] },
     "in_progress_merge_commit_sha": { "type": ["string", "null"] },
-    "locked_at": { "type": ["string", "null"] },
     "merge_error": { "type": ["string", "null"] },
     "merge_commit_sha": { "type": ["string", "null"] },
     "merge_params": { "type": ["object", "null"] },
@@ -94,7 +93,8 @@
     "commit_change_content_path": { "type": "string" },
     "remove_wip_path": { "type": "string" },
     "commits_count": { "type": "integer" },
-    "remove_source_branch": { "type": ["boolean", "null"] }
+    "remove_source_branch": { "type": ["boolean", "null"] },
+    "merge_ongoing": { "type": "boolean" }
   },
   "additionalProperties": false
 }