Merge branch 'drop-satellites'

CHANGELOG

@@ -32,6 +32,8 @@ v 7.14.0 (unreleased)
   - Mention group and project name in creation, update and deletion notices (Achilleas Pipinellis)
   - Remove redis-store TTL monkey patch
   - Add support for CI skipped status
+  - Fetch code from forks to refs/merge-requests/:id/head when merge request created
+  - Remove satellites 
 v 7.13.2
   - Fix randomly failed spec

Gemfile.lock

@@ -875,4 +875,4 @@ DEPENDENCIES
   wikicloth (= 0.8.1)
 BUNDLED WITH
-   1.10.5
+   1.10.4

app/assets/javascripts/merge_request_widget.js.coffee

@@ -19,7 +19,7 @@ class @MergeRequestWidget
           when 'merged'
             location.reload()
           else
-            setTimeout(merge_request_widget.mergeInProgress, 3000)
+            setTimeout(merge_request_widget.mergeInProgress, 2000)
       dataType: 'json'
   getMergeStatus: ->

app/controllers/projects/compare_controller.rb

@@ -13,13 +13,8 @@ class Projects::CompareController < Projects::ApplicationController
     base_ref = Addressable::URI.unescape(params[:from])
     @ref = head_ref = Addressable::URI.unescape(params[:to])
-    compare_result = CompareService.new.execute(
-      current_user,
-      @project,
-      head_ref,
-      @project,
-      base_ref
-    )
+    compare_result = CompareService.new.
+      execute(@project, head_ref, @project, base_ref)
     @commits = compare_result.commits
     @diffs = compare_result.diffs

app/controllers/projects/merge_requests_controller.rb

-require 'gitlab/satellite/satellite'
-
 class Projects::MergeRequestsController < Projects::ApplicationController
   before_action :module_enabled
   before_action :merge_request, only: [
-    :edit, :update, :show, :diffs, :commits, :automerge, :automerge_check,
+    :edit, :update, :show, :diffs, :commits, :merge, :merge_check,
     :ci_status, :toggle_subscription
   ]
   before_action :closes_issues, only: [:edit, :update, :show, :diffs, :commits]
@@ -137,7 +135,7 @@ class Projects::MergeRequestsController < Projects::ApplicationController
     end
   end
-  def automerge_check
+  def merge_check
     if @merge_request.unchecked?
       @merge_request.check_if_can_be_merged
     end
@@ -147,11 +145,11 @@ class Projects::MergeRequestsController < Projects::ApplicationController
     render partial: "projects/merge_requests/widget/show.html.haml", layout: false
   end
-  def automerge
+  def merge
     return access_denied! unless @merge_request.can_be_merged_by?(current_user)
-    if @merge_request.automergeable?
-      AutoMergeWorker.perform_async(@merge_request.id, current_user.id, params)
+    if @merge_request.mergeable?
+      MergeWorker.perform_async(@merge_request.id, current_user.id, params)
       @status = true
     else
       @status = false

app/models/merge_request.rb

@@ -41,8 +41,6 @@ class MergeRequest < ActiveRecord::Base
   delegate :commits, :diffs, :last_commit, :last_commit_short_sha, to: :merge_request_diff, prefix: nil
-  attr_accessor :should_remove_source_branch
-
   # When this attribute is true some MR validation is ignored
   # It allows us to close or modify broken merge requests
   attr_accessor :allow_broken
@@ -57,7 +55,7 @@ class MergeRequest < ActiveRecord::Base
       transition [:reopened, :opened] => :closed
     end
-    event :merge do
+    event :mark_as_merged do
       transition [:reopened, :opened, :locked] => :merged
     end
@@ -206,11 +204,7 @@ class MergeRequest < ActiveRecord::Base
   def check_if_can_be_merged
     can_be_merged =
-      if for_fork?
-        Gitlab::Satellite::MergeAction.new(self.author, self).can_be_merged?
-      else
-        project.repository.can_be_merged?(source_branch, target_branch)
-      end
+      project.repository.can_be_merged?(source_sha, target_branch)
     if can_be_merged
       mark_as_mergeable
@@ -227,18 +221,6 @@ class MergeRequest < ActiveRecord::Base
     self.target_project.events.where(target_id: self.id, target_type: "MergeRequest", action: Event::CLOSED).last
   end
-  def automerge!(current_user, commit_message = nil)
-    return unless automergeable?
-
-    MergeRequests::AutoMergeService.
-      new(target_project, current_user).
-      execute(self, commit_message)
-  end
-
-  def remove_source_branch?
-    self.should_remove_source_branch && !self.source_project.root_ref?(self.source_branch) && !self.for_fork?
-  end
-
   def open?
     opened? || reopened?
   end
@@ -247,11 +229,11 @@ class MergeRequest < ActiveRecord::Base
     title =~ /\A\[?WIP\]?:? /i
   end
-  def automergeable?
+  def mergeable?
     open? && !work_in_progress? && can_be_merged?
   end
-  def automerge_status
+  def gitlab_merge_status
     if work_in_progress?
       "work_in_progress"
     else
@@ -278,14 +260,14 @@ class MergeRequest < ActiveRecord::Base
   #
   # see "git diff"
   def to_diff(current_user)
-    Gitlab::Satellite::MergeAction.new(current_user, self).diff_in_satellite
+    target_project.repository.diff_text(target_branch, source_sha)
   end
   # Returns the commit as a series of email patches.
   #
   # see "git format-patch"
   def to_patch(current_user)
-    Gitlab::Satellite::MergeAction.new(current_user, self).format_patch
+    target_project.repository.format_patch(target_branch, source_sha)
   end
   def hook_attrs
@@ -436,4 +418,30 @@ class MergeRequest < ActiveRecord::Base
       "Open"
     end
   end
+
+  def target_sha
+    @target_sha ||= target_project.
+      repository.commit(target_branch).sha
+  end
+
+  def source_sha
+    commits.first.sha
+  end
+
+  def fetch_ref
+    target_project.repository.fetch_ref(
+      source_project.repository.path_to_repo,
+      "refs/heads/#{source_branch}",
+      "refs/merge-requests/#{id}/head"
+    )
+  end
+
+  def in_locked_state
+    begin
+      lock_mr
+      yield
+    ensure
+      unlock_mr if locked?
+    end
+  end
 end

app/models/merge_request_diff.rb

@@ -16,9 +16,8 @@ require Rails.root.join("app/models/commit")
 class MergeRequestDiff < ActiveRecord::Base
   include Sortable
-  # Prevent store of diff
-  # if commits amount more then 200
-  COMMITS_SAFE_SIZE = 200
+  # Prevent store of diff if commits amount more then 500
+  COMMITS_SAFE_SIZE = 500
   attr_reader :commits, :diffs
@@ -124,12 +123,12 @@ class MergeRequestDiff < ActiveRecord::Base
     if new_diffs.any?
       if new_diffs.size > Commit::DIFF_HARD_LIMIT_FILES
         self.state = :overflow_diff_files_limit
-        new_diffs = []
+        new_diffs = new_diffs.first[Commit::DIFF_HARD_LIMIT_LINES]
       end
       if new_diffs.sum { |diff| diff.diff.lines.count } > Commit::DIFF_HARD_LIMIT_LINES
         self.state = :overflow_diff_lines_limit
-        new_diffs = []
+        new_diffs = new_diffs.first[Commit::DIFF_HARD_LIMIT_LINES]
       end
     end
@@ -160,12 +159,21 @@ class MergeRequestDiff < ActiveRecord::Base
   private
   def compare_result
-    @compare_result ||= CompareService.new.execute(
-      merge_request.author,
-      merge_request.source_project,
-      merge_request.source_branch,
-      merge_request.target_project,
-      merge_request.target_branch,
-    )
+    @compare_result ||=
+      begin
+        # Update ref if merge request is from fork
+        merge_request.fetch_ref if merge_request.for_fork?
+
+        # Get latest sha of branch from source project
+        source_sha = merge_request.source_project.commit(source_branch).sha
+
+        Gitlab::CompareResult.new(
+          Gitlab::Git::Compare.new(
+            merge_request.target_project.repository.raw_repository,
+            merge_request.target_branch,
+            source_sha,
+          )
+        )
+      end
   end
 end

app/models/namespace.rb

@@ -115,12 +115,11 @@ class Namespace < ActiveRecord::Base
   def move_dir
     if gitlab_shell.mv_namespace(path_was, path)
-      # If repositories moved successfully we need to remove old satellites
-      # and send update instructions to users.
+      # If repositories moved successfully we need to
+      # send update instructions to users.
       # However we cannot allow rollback since we moved namespace dir
       # So we basically we mute exceptions in next actions
       begin
-        gitlab_shell.rm_satellites(path_was)
         send_update_instructions
       rescue
         # Returning false does not rollback after_* transaction but gives

app/models/project.rb

@@ -520,14 +520,6 @@ class Project < ActiveRecord::Base
     !repository.exists? || repository.empty?
   end
-  def ensure_satellite_exists
-    self.satellite.create unless self.satellite.exists?
-  end
-
-  def satellite
-    @satellite ||= Gitlab::Satellite::Satellite.new(self)
-  end
-
   def repo
     repository.raw
   end
@@ -597,14 +589,11 @@ class Project < ActiveRecord::Base
     new_path_with_namespace = File.join(namespace_dir, path)
     if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
-      # If repository moved successfully we need to remove old satellite
-      # and send update instructions to users.
+      # If repository moved successfully we need to send update instructions to users.
       # However we cannot allow rollback since we moved repository
       # So we basically we mute exceptions in next actions
       begin
         gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
-        gitlab_shell.rm_satellites(old_path_with_namespace)
-        ensure_satellite_exists
         send_move_instructions
         reset_events_cache
       rescue
@@ -702,7 +691,6 @@ class Project < ActiveRecord::Base
   def create_repository
     if forked?
       if gitlab_shell.fork_repository(forked_from_project.path_with_namespace, self.namespace.path)
-        ensure_satellite_exists
         true
       else
         errors.add(:base, 'Failed to fork repository via gitlab-shell')

app/models/project_services/gitlab_ci_service.rb

@@ -74,6 +74,8 @@ class GitlabCiService < CiService
     else
       :error
     end
+  rescue Errno::ECONNREFUSED
+    :error
   end
   def fork_registration(new_project, private_token)
@@ -103,6 +105,8 @@ class GitlabCiService < CiService
     if response.code == 200 and response["coverage"]
       response["coverage"]
     end
+  rescue Errno::ECONNREFUSED
+    nil
   end
   def build_page(sha, ref)

app/models/repository.rb

@@ -411,15 +411,36 @@ class Repository
     }
   end
-  def can_be_merged?(source_branch, target_branch)
+  def can_be_merged?(source_sha, target_branch)
     our_commit = rugged.branches[target_branch].target
-    their_commit = rugged.branches[source_branch].target
+    their_commit = rugged.lookup(source_sha)
     if our_commit && their_commit
       !rugged.merge_commits(our_commit, their_commit).conflicts?
+    else
+      false
     end
   end
+  def merge(source_sha, target_branch, options = {})
+    our_commit = rugged.branches[target_branch].target
+    their_commit = rugged.lookup(source_sha)
+
+    raise "Invalid merge target" if our_commit.nil?
+    raise "Invalid merge source" if their_commit.nil?
+
+    merge_index = rugged.merge_commits(our_commit, their_commit)
+    return false if merge_index.conflicts?
+
+    actual_options = options.merge(
+      parents: [our_commit, their_commit],
+      tree: merge_index.write_tree(rugged),
+      update_ref: "refs/heads/#{target_branch}"
+    )
+
+    Rugged::Commit.create(rugged, actual_options)
+  end
+
   def search_files(query, ref)
     offset = 2
     args = %W(git grep -i -n --before-context #{offset} --after-context #{offset} #{query} #{ref || root_ref})
@@ -453,6 +474,11 @@ class Repository
     )
   end
+  def fetch_ref(source_path, source_ref, target_ref)
+    args = %W(git fetch #{source_path} #{source_ref}:#{target_ref})
+    Gitlab::Popen.popen(args, path_to_repo)
+  end
+
   private
   def cache

app/services/base_service.rb

@@ -31,6 +31,10 @@ class BaseService
     SystemHooksService.new
   end
+  def repository
+    project.repository
+  end
+
   # Add an error to the specified model for restricted visibility levels
   def deny_visibility_level(model, denied_visibility_level = nil)
     denied_visibility_level ||= model.visibility_level

app/services/compare_service.rb

+require 'securerandom'
+
 # Compare 2 branches for one repo or between repositories
 # and return Gitlab::CompareResult object that responds to commits and diffs
 class CompareService
-  def execute(current_user, source_project, source_branch, target_project, target_branch)
-    # Try to compare branches to get commits list and diffs
-    #
-    # Note: Use satellite only when need to compare between two repos
-    # because satellites are slower than operations on bare repo
-    if target_project == source_project
-      Gitlab::CompareResult.new(
-        Gitlab::Git::Compare.new(
-          target_project.repository.raw_repository,
-          target_branch,
-          source_branch,
-        )
+  def execute(source_project, source_branch, target_project, target_branch)
+    source_sha = source_project.commit(source_branch).sha
+
+    # If compare with other project we need to fetch ref first
+    unless target_project == source_project
+      random_string = SecureRandom.hex
+
+      target_project.repository.fetch_ref(
+        source_project.repository.path_to_repo,
+        "refs/heads/#{source_branch}",
+        "refs/tmp/#{random_string}/head"
       )
-    else
-      Gitlab::Satellite::CompareAction.new(
-        current_user,
-        target_project,
-        target_branch,
-        source_project,
-        source_branch
-      ).result
     end
+
+    Gitlab::CompareResult.new(
+      Gitlab::Git::Compare.new(
+        target_project.repository.raw_repository,
+        target_branch,
+        source_sha,
+      )
+    )
   end
 end

app/services/files/base_service.rb

@@ -33,15 +33,8 @@ module Files
     private
-    def repository
-      project.repository
-    end
-
     def after_commit(sha, branch)
-      commit = repository.commit(sha)
-      full_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{branch}"
-      old_sha = commit.parent_id || Gitlab::Git::BLANK_SHA
-      GitPushService.new.execute(project, current_user, old_sha, sha, full_ref)
+      PostCommitService.new(project, current_user).execute(sha, branch)
     end
     def current_branch

app/services/git_push_service.rb

@@ -10,16 +10,14 @@ class GitPushService
   #
   # Next, this method:
   #  1. Creates the push event
-  #  2. Ensures that the project satellite exists
-  #  3. Updates merge requests
-  #  4. Recognizes cross-references from commit messages
-  #  5. Executes the project's web hooks
-  #  6. Executes the project's services
+  #  2. Updates merge requests
+  #  3. Recognizes cross-references from commit messages
+  #  4. Executes the project's web hooks
+  #  5. Executes the project's services
   #
   def execute(project, user, oldrev, newrev, ref)
     @project, @user = project, user
-    project.ensure_satellite_exists
     project.repository.expire_cache
     if push_remove_branch?(ref, newrev)

app/services/merge_requests/auto_merge_service.rb

-module MergeRequests
-  # AutoMergeService class
-  #
-  # Do git merge in satellite and in case of success
-  # mark merge request as merged and execute all hooks and notifications
-  # Called when you do merge via GitLab UI
-  class AutoMergeService < BaseMergeService
-    attr_reader :merge_request, :commit_message
-
-    def execute(merge_request, commit_message)
-      @commit_message = commit_message
-      @merge_request = merge_request
-
-      merge_request.lock_mr
-
-      if merge!
-        merge_request.merge
-        create_merge_event(merge_request, current_user)
-        create_note(merge_request)
-        notification_service.merge_mr(merge_request, current_user)
-        execute_hooks(merge_request, 'merge')
-        true
-      else
-        merge_request.unlock_mr
-        false
-      end
-    rescue
-      merge_request.unlock_mr if merge_request.locked?
-      merge_request.mark_as_unmergeable
-      false
-    end
-
-    def merge!
-      if merge_request.for_fork?
-        Gitlab::Satellite::MergeAction.new(current_user, merge_request).merge!(commit_message)
-      else
-        # Merge local branches using rugged instead of satellites
-        if sha = commit
-          after_commit(sha, merge_request.target_branch)
-
-          if merge_request.remove_source_branch?
-            DeleteBranchService.new(merge_request.source_project, current_user).execute(merge_request.source_branch)
-          end
-
-          true
-        else
-          false
-        end
-      end
-    end
-
-    def commit
-      committer = repository.user_to_comitter(current_user)
-
-      options = {
-        message: commit_message,
-        author: committer,
-        committer: committer
-      }
-
-      repository.merge(merge_request.source_branch, merge_request.target_branch, options)
-    end
-
-    def after_commit(sha, branch)
-      commit = repository.commit(sha)
-      full_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{branch}"
-      old_sha = commit.parent_id || Gitlab::Git::BLANK_SHA
-      GitPushService.new.execute(project, current_user, old_sha, sha, full_ref)
-    end
-
-    def repository
-      project.repository
-    end
-  end
-end

app/services/merge_requests/base_merge_service.rb

-module MergeRequests
-  class BaseMergeService < MergeRequests::BaseService
-
-    private
-
-    def create_merge_event(merge_request, current_user)
-      EventCreateService.new.merge_mr(merge_request, current_user)
-    end
-  end
-end

app/services/merge_requests/build_service.rb

@@ -12,12 +12,16 @@ module MergeRequests
       merge_request.target_project ||= (project.forked_from_project || project)
       merge_request.target_branch ||= merge_request.target_project.default_branch
-      unless merge_request.target_branch && merge_request.source_branch
-        return build_failed(merge_request, nil)
+      if merge_request.target_branch.blank? || merge_request.source_branch.blank?
+        message =
+          if params[:source_branch] || params[:target_branch]
+            "You must select source and target branch"
+          end
+
+        return build_failed(merge_request, message)
       end
       compare_result = CompareService.new.execute(
-        current_user,
         merge_request.source_project,
         merge_request.source_branch,
         merge_request.target_project,
@@ -40,7 +44,6 @@ module MergeRequests
           merge_request.compare_diffs = diffs
         elsif diffs == false
-          # satellite timeout return false
           merge_request.can_be_created = false
           merge_request.compare_failed = true
         end
@@ -59,9 +62,6 @@ module MergeRequests
       end
       merge_request
-
-    rescue Gitlab::Satellite::BranchesWithoutParent
-      return build_failed(merge_request, "Selected branches have no common commit so they cannot be merged.")
     end
     def build_failed(merge_request, message)

app/services/merge_requests/merge_service.rb

 module MergeRequests
   # MergeService class
   #
-  # Mark existing merge request as merged
-  # and execute all hooks and notifications
-  # Called when you do merge via command line and push code
-  # to target branch
-  class MergeService < BaseMergeService
+  # Do git merge and in case of success
+  # mark merge request as merged and execute all hooks and notifications
+  # Executed when you do merge via GitLab UI
+  #
+  class MergeService < MergeRequests::BaseService
+    attr_reader :merge_request, :commit_message
+
     def execute(merge_request, commit_message)
-      merge_request.merge
-      create_merge_event(merge_request, current_user)
-      create_note(merge_request)
-      notification_service.merge_mr(merge_request, current_user)
-      execute_hooks(merge_request, 'merge')
-      true
-    rescue
-      false
+      @commit_message = commit_message
+      @merge_request = merge_request
+      unless @merge_request.mergeable?
+        return error('Merge request is not mergeable')
+      end
+
+      merge_request.in_locked_state do
+        if merge_changes
+          after_merge
+          success
+        else
+          error('Can not merge changes')
+        end
+      end
+    end
+
+    private
+
+    def merge_changes
+      if sha = commit
+        after_commit(sha, merge_request.target_branch)
+      end
+    end
+
+    def commit
+      committer = repository.user_to_comitter(current_user)
+
+      options = {
+        message: commit_message,
+        author: committer,
+        committer: committer
+      }
+
+      repository.merge(merge_request.source_sha, merge_request.target_branch, options)
+    end
+
+    def after_commit(sha, branch)
+      PostCommitService.new(project, current_user).execute(sha, branch)
+    end
+    def after_merge
+      MergeRequests::PostMergeService.new(project, current_user).execute(merge_request)
     end
   end
 end

app/services/merge_requests/post_merge_service.rb

+module MergeRequests
+  # PostMergeService class
+  #
+  # Mark existing merge request as merged
+  # and execute all hooks and notifications
+  #
+  class PostMergeService < MergeRequests::BaseService
+    def execute(merge_request)
+      merge_request.mark_as_merged
+      create_merge_event(merge_request, current_user)
+      create_note(merge_request)
+      notification_service.merge_mr(merge_request, current_user)
+      execute_hooks(merge_request, 'merge')
+    end
+
+    private
+
+    def create_merge_event(merge_request, current_user)
+      EventCreateService.new.merge_mr(merge_request, current_user)
+    end
+  end
+end