Merge branch 'issue_116' into 'master'

Add ability to push to remote repositories

Closes #116 

See merge request !249

CHANGELOG

@@ -33,6 +33,7 @@ v 8.7.0 (unreleased)
   - API: Expose user location (Robert Schilling)
   - ClosingIssueExtractor regex now also works with colons. e.g. "Fixes: #1234" !3591
   - Update number of Todos in the sidebar when it's marked as "Done". !3600
+  - Add ability to sync to remote mirrors. !249
 v 8.6.5
   - Fix importing from GitHub Enterprise. !3529

CHANGELOG-EE

@@ -4,6 +4,7 @@ v 8.7.0 (unreleased)
   - Update GitLab Pages to 0.2.1: support user-defined 404 pages
   - Refactor group sync to pull access level logic to its own class. !306
   - [Elastic] Stabilize database indexer if database is inconsistent
+  - Add ability to sync to remote mirrors. !249
 v 8.6.6
   - Fix LDAP group sync regression for groups with member value `uid=<username>` !335

app/assets/stylesheets/pages/projects.scss

@@ -461,7 +461,7 @@ pre.light-well {
 .cannot-be-merged,
 .cannot-be-merged:hover {
-  color: #e62958;
+  color: $error-exclamation-point;
   margin-top: 2px;
 }
@@ -474,3 +474,7 @@ pre.light-well {
     color: #fff;
   }
 }
+
+.disabled-item {
+  @extend .btn.disabled;
+}

app/controllers/projects/mirrors_controller.rb

@@ -2,6 +2,7 @@ class Projects::MirrorsController < Projects::ApplicationController
   # Authorize
   before_action :authorize_admin_project!, except: [:update_now]
   before_action :authorize_push_code!, only: [:update_now]
+  before_action :remote_mirror, only: [:show, :update]
   layout "project_settings"
@@ -28,15 +29,25 @@ class Projects::MirrorsController < Projects::ApplicationController
   end
   def update_now
-    @project.update_mirror
-    flash[:notice] = "The repository is being updated..."
+    if params[:sync_remote]
+      @project.update_remote_mirrors
+      flash[:notice] = "The remote repository is being updated..."
+    else
+      @project.update_mirror
+      flash[:notice] = "The repository is being updated..."
+    end
     redirect_back_or_default(default: namespace_project_path(@project.namespace, @project))
   end
   private
+  def remote_mirror
+    @remote_mirror = @project.remote_mirrors.first_or_initialize
+  end
+
   def mirror_params
-    params.require(:project).permit(:mirror, :import_url, :mirror_user_id, :mirror_trigger_builds)
+    params.require(:project).permit(:mirror, :import_url, :mirror_user_id, :mirror_trigger_builds,
+      remote_mirrors_attributes: [:url, :id, :enabled])
   end
 end

app/models/project.rb

@@ -2,41 +2,54 @@
 #
 # Table name: projects
 #
-#  id                     :integer          not null, primary key
-#  name                   :string(255)
-#  path                   :string(255)
-#  description            :text
-#  created_at             :datetime
-#  updated_at             :datetime
-#  creator_id             :integer
-#  issues_enabled         :boolean          default(TRUE), not null
-#  wall_enabled           :boolean          default(TRUE), not null
-#  merge_requests_enabled :boolean          default(TRUE), not null
-#  wiki_enabled           :boolean          default(TRUE), not null
-#  namespace_id           :integer
-#  issues_tracker         :string(255)      default("gitlab"), not null
-#  issues_tracker_id      :string(255)
-#  snippets_enabled       :boolean          default(TRUE), not null
-#  last_activity_at       :datetime
-#  import_url             :string(255)
-#  visibility_level       :integer          default(0), not null
-#  archived               :boolean          default(FALSE), not null
-#  avatar                 :string(255)
-#  import_status          :string(255)
-#  repository_size        :float            default(0.0)
-#  star_count             :integer          default(0), not null
-#  import_type            :string(255)
-#  import_source          :string(255)
-#  commit_count           :integer          default(0)
-#  import_error           :text
-#  ci_id                  :integer
-#  builds_enabled         :boolean          default(TRUE), not null
-#  shared_runners_enabled :boolean          default(TRUE), not null
-#  runners_token          :string
-#  build_coverage_regex   :string
-#  build_allow_git_fetch  :boolean          default(TRUE), not null
-#  build_timeout          :integer          default(3600), not null
-#  pending_delete         :boolean
+#  id                               :integer          not null, primary key
+#  name                             :string(255)
+#  path                             :string(255)
+#  description                      :text
+#  created_at                       :datetime
+#  updated_at                       :datetime
+#  creator_id                       :integer
+#  issues_enabled                   :boolean          default(TRUE), not null
+#  wall_enabled                     :boolean          default(TRUE), not null
+#  merge_requests_enabled           :boolean          default(TRUE), not null
+#  wiki_enabled                     :boolean          default(TRUE), not null
+#  namespace_id                     :integer
+#  issues_tracker                   :string(255)      default("gitlab"), not null
+#  issues_tracker_id                :string(255)
+#  snippets_enabled                 :boolean          default(TRUE), not null
+#  last_activity_at                 :datetime
+#  import_url                       :string(255)
+#  visibility_level                 :integer          default(0), not null
+#  archived                         :boolean          default(FALSE), not null
+#  avatar                           :string(255)
+#  import_status                    :string(255)
+#  repository_size                  :float            default(0.0)
+#  star_count                       :integer          default(0), not null
+#  import_type                      :string(255)
+#  import_source                    :string(255)
+#  commit_count                     :integer          default(0)
+#  import_error                     :text
+#  ci_id                            :integer
+#  builds_enabled                   :boolean          default(TRUE), not null
+#  shared_runners_enabled           :boolean          default(TRUE), not null
+#  runners_token                    :string
+#  build_coverage_regex             :string
+#  build_allow_git_fetch            :boolean          default(TRUE), not null
+#  build_timeout                    :integer          default(3600), not null
+#  pending_delete                   :boolean          default(FALSE)
+#  public_builds                    :boolean          default(TRUE), not null
+#  merge_requests_template          :text
+#  merge_requests_rebase_enabled    :boolean          default(FALSE)
+#  approvals_before_merge           :integer          default(0), not null
+#  reset_approvals_on_push          :boolean          default(TRUE)
+#  merge_requests_ff_only_enabled   :boolean          default(FALSE)
+#  issues_template                  :text
+#  mirror                           :boolean          default(FALSE), not null
+#  mirror_last_update_at            :datetime
+#  mirror_last_successful_update_at :datetime
+#  mirror_user_id                   :integer
+#  mirror_trigger_builds            :boolean          default(FALSE), not null
+#  main_language                    :string
 #
 require 'carrierwave/orm/activerecord'
@@ -76,18 +89,8 @@ class Project < ActiveRecord::Base
   after_destroy :remove_pages
-  # update visibility_level of forks
   after_update :update_forks_visibility_level
-  def update_forks_visibility_level
-    return unless visibility_level < visibility_level_was
-
-    forks.each do |forked_project|
-      if forked_project.visibility_level > visibility_level
-        forked_project.visibility_level = visibility_level
-        forked_project.save!
-      end
-    end
-  end
+  after_update :remove_mirror_repository_reference, if: :import_url_changed?
   ActsAsTaggableOn.strict_case_match = true
   acts_as_taggable_on :tags
@@ -175,8 +178,11 @@ class Project < ActiveRecord::Base
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, dependent: :destroy, class_name: 'Ci::Variable', foreign_key: :gl_project_id
   has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :gl_project_id
+  has_many :remote_mirrors, dependent: :destroy
   accepts_nested_attributes_for :variables, allow_destroy: true
+  accepts_nested_attributes_for :remote_mirrors,
+    allow_destroy: true, reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? }
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :members, to: :team, prefix: true
@@ -204,6 +210,7 @@ class Project < ActiveRecord::Base
     url: { protocols: %w(ssh git http https) },
     if: :external_import?
   validates :import_url, presence: true, if: :mirror?
+  validate  :import_url_availability, if: :import_url_changed?
   validates :mirror_user, presence: true, if: :mirror?
   validates :star_count, numericality: { greater_than_or_equal_to: 0 }
   validate :check_limit, on: :create
@@ -216,6 +223,7 @@ class Project < ActiveRecord::Base
   add_authentication_token_field :runners_token
   before_save :ensure_runners_token
+  before_validation :mark_remote_mirrors_for_removal
   mount_uploader :avatar, AvatarUploader
@@ -236,6 +244,7 @@ class Project < ActiveRecord::Base
   scope :non_archived, -> { where(archived: false) }
   scope :mirror, -> { where(mirror: true) }
   scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct }
+  scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
   state_machine :import_status, initial: :none do
     event :import_start do
@@ -524,6 +533,23 @@ class Project < ActiveRecord::Base
     end
   end
+  def mark_import_as_failed(error_message)
+    import_fail
+    update_column(:import_error, error_message)
+  end
+
+  def has_remote_mirror?
+    remote_mirrors.enabled.exists?
+  end
+
+  def updating_remote_mirror?
+    remote_mirrors.enabled.started.exists?
+  end
+
+  def update_remote_mirrors
+    remote_mirrors.each(&:sync)
+  end
+
   def fetch_mirror
     return unless mirror?
@@ -1184,4 +1210,31 @@ class Project < ActiveRecord::Base
   def ff_merge_must_be_possible?
     self.merge_requests_ff_only_enabled || self.merge_requests_rebase_enabled
   end
+
+  private
+
+  def update_forks_visibility_level
+    return unless visibility_level < visibility_level_was
+
+    forks.each do |forked_project|
+      if forked_project.visibility_level > visibility_level
+        forked_project.visibility_level = visibility_level
+        forked_project.save!
+      end
+    end
+  end
+
+  def remove_mirror_repository_reference
+    repository.remove_remote(Repository::MIRROR_REMOTE)
+  end
+
+  def import_url_availability
+    if remote_mirrors.find_by(url: import_url)
+      errors.add(:import_url, 'is already in use by a remote mirror')
+    end
+  end
+
+  def mark_remote_mirrors_for_removal
+    remote_mirrors.each(&:mark_for_delete_if_blank_url)
+  end
 end

app/models/remote_mirror.rb

+# == Schema Information
+#
+# Table name: remote_mirrors
+#
+#  id                        :integer          not null, primary key
+#  project_id                :integer
+#  url                       :string
+#  last_update_at            :datetime
+#  last_error                :string
+#  created_at                :datetime         not null
+#  updated_at                :datetime         not null
+#  last_successful_update_at :datetime
+#  update_status             :string
+#  enabled                   :boolean          default(TRUE)
+#
+
+class RemoteMirror < ActiveRecord::Base
+  include AfterCommitQueue
+
+  attr_encrypted :credentials,
+                 key: Gitlab::Application.secrets.db_key_base,
+                 marshal: true,
+                 encode: true,
+                 mode: :per_attribute_iv_and_salt
+
+  belongs_to :project
+
+  validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
+  validate  :url_availability, if: :url_changed?
+
+  after_save :refresh_remote, if: :url_changed?
+  after_update :reset_fields, if: :url_changed?
+  after_destroy :remove_remote
+
+  scope :enabled, -> { where(enabled: true) }
+  scope :started, -> { with_update_status(:started) }
+  scope :stuck,   -> { started.where('last_update_at < ?', 1.day.ago) }
+
+  state_machine :update_status, initial: :none do
+    event :update_start do
+      transition [:none, :finished] => :started
+    end
+
+    event :update_finish do
+      transition started: :finished
+    end
+
+    event :update_fail do
+      transition started: :failed
+    end
+
+    event :update_retry do
+      transition failed: :started
+    end
+
+    state :started
+    state :finished
+    state :failed
+
+    after_transition any => :started, do: :schedule_update_job
+
+    after_transition started: :finished do |remote_mirror, transaction|
+      timestamp = Time.now
+      remote_mirror.update_attributes!(
+        last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
+      )
+    end
+
+    after_transition started: :failed do |remote_mirror, transaction|
+      remote_mirror.update(last_update_at: Time.now)
+    end
+  end
+
+  def ref_name
+    "remote_mirror_#{id}"
+  end
+
+  def update_failed?
+    update_status == 'failed'
+  end
+
+  def update_in_progress?
+    update_status == 'started'
+  end
+
+  def sync
+    return if !enabled || update_in_progress?
+
+    update_failed? ? update_retry : update_start
+  end
+
+  def mark_for_delete_if_blank_url
+    mark_for_destruction if url.blank?
+  end
+
+  def mark_as_failed(error_message)
+    update_fail
+    update_column(:last_error, error_message)
+  end
+
+  def url=(value)
+    mirror_url = Gitlab::ImportUrl.new(value)
+    self.credentials = mirror_url.credentials
+
+    super(mirror_url.sanitized_url)
+  end
+
+  def url
+    if super
+      Gitlab::ImportUrl.new(super, credentials: credentials).full_url
+    end
+  end
+
+  def safe_url
+    return if url.nil?
+
+    result = URI.parse(url)
+    result.password = '*****' if result.password
+    result.user = '*****' if result.user && result.user != "git" #tokens or other data may be saved as user
+    result.to_s
+  end
+
+  private
+
+  def url_availability
+    if project.import_url == url
+      errors.add(:url, 'is already in use')
+    end
+  end
+
+  def reset_fields
+    update_columns(
+      last_error: nil,
+      last_update_at: nil,
+      last_successful_update_at: nil,
+      update_status: 'finished'
+    )
+  end
+
+  def schedule_update_job
+    run_after_commit(:add_update_job)
+  end
+
+  def add_update_job
+    if project.repository_exists?
+      RepositoryUpdateRemoteMirrorWorker.perform_async(self.id)
+    end
+  end
+
+  def refresh_remote
+    project.repository.remove_remote(ref_name)
+    project.repository.add_remote(ref_name, url)
+  end
+
+  def remove_remote
+    project.repository.remove_remote(ref_name)
+  end
+end

app/models/repository.rb

 require 'securerandom'
+require 'forwardable'
 class Repository
   include Elastic::RepositoriesSearch
@@ -16,6 +17,8 @@ class Repository
   attr_accessor :path_with_namespace, :project
+  delegate :push_remote_branches, :delete_remote_branches, to: :gitlab_shell
+
   def self.clean_old_archives
     repository_downloads_path = Gitlab.config.gitlab.repository_downloads_path
@@ -189,6 +192,13 @@ class Repository
     raw_repository.remote_update(name, url: url)
   end
+  def remove_remote(name)
+    raw_repository.remote_delete(name)
+    true
+  rescue Rugged::ConfigError
+    false
+  end
+
   def set_remote_as_mirror(name)
     remote_config = raw_repository.rugged.config
@@ -198,8 +208,14 @@ class Repository
     remote_config["remote.#{name}.prune"] = true
   end
-  def fetch_remote(remote)
-    gitlab_shell.fetch_remote(path_with_namespace, remote)
+  def fetch_remote(remote, forced: false, no_tags: false)
+    gitlab_shell.fetch_remote(path_with_namespace, remote, forced: forced, no_tags: no_tags)
+  end
+
+  def remote_tags(remote)
+    gitlab_shell.list_remote_tags(path_with_namespace, remote).map do |name, target|
+      Gitlab::Git::Tag.new(name, target)
+    end
   end
   def fetch_remote_forced!(remote)
@@ -667,6 +683,22 @@ class Repository
   alias_method :branches, :local_branches
+  def remote_branches(remote_name)
+    branches = []
+
+    rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
+      name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
+
+      begin
+        branches << Gitlab::Git::Branch.new(name, ref.target)
+      rescue Rugged::ReferenceError
+        # Omit invalid branch
+      end
+    end
+
+    branches
+  end
+
   def tags
     @tags ||= raw_repository.tags
   end
@@ -842,15 +874,7 @@ class Repository
   end
   def upstream_branches
-    rugged.references.each("refs/remotes/#{Repository::MIRROR_REMOTE}/*").map do |ref|
-      name = ref.name.sub(/\Arefs\/remotes\/#{Repository::MIRROR_REMOTE}\//, "")
-
-      begin
-        Gitlab::Git::Branch.new(name, ref.target)
-      rescue Rugged::ReferenceError
-        # Omit invalid branch
-      end
-    end.compact
+    @upstream_branches ||= remote_branches(Repository::MIRROR_REMOTE)
   end
   def diverged_from_upstream?(branch_name)
@@ -864,6 +888,17 @@ class Repository
     end
   end
+  def upstream_has_diverged?(branch_name, remote_ref)
+    branch_commit = commit(branch_name)
+    upstream_commit = commit("refs/remotes/#{remote_ref}/#{branch_name}")
+
+    if upstream_commit
+      !is_ancestor?(upstream_commit.id, branch_commit.id)
+    else
+      false
+    end
+  end
+
   def up_to_date_with_upstream?(branch_name)
     branch_commit = commit(branch_name)
     upstream_commit = commit("refs/remotes/#{MIRROR_REMOTE}/#{branch_name}")

app/services/projects/update_remote_mirror_service.rb

+module Projects
+  class UpdateRemoteMirrorService < BaseService
+    attr_reader :mirror, :errors
+
+    def execute(remote_mirror)
+      @mirror = remote_mirror
+      @errors = []
+
+      begin
+        repository.fetch_remote(mirror.ref_name, no_tags: true)
+
+        if divergent_branches.present?
+          errors << "The following branches have diverged from their local counterparts: #{divergent_branches.to_sentence}"
+        end
+
+        push_branches if changed_branches.present?
+        delete_branches if deleted_branches.present?
+
+        push_tags if changed_tags.present?
+        delete_tags if deleted_tags.present?
+      rescue Gitlab::Shell::Error => e
+        errors << e.message.strip
+      end
+
+      if errors.present?
+        error(errors.join("\n\n"))
+      else
+        success
+      end
+    end
+
+    private
+
+    def local_branches
+      @local_branches ||= repository.local_branches.each_with_object({}) do |branch, branches|
+        branches[branch.name] = branch
+      end
+    end
+
+    def remote_branches
+      @remote_branches ||= repository.remote_branches(mirror.ref_name).each_with_object({}) do |branch, branches|
+        branches[branch.name] = branch
+      end
+    end
+
+    def push_branches
+      default_branch, branches = changed_branches.partition { |name| project.default_branch == name }
+
+      # Push the default branch first so it works fine when remote mirror is empty.
+      branches.unshift(*default_branch)
+
+      repository.push_remote_branches(project.path_with_namespace, mirror.ref_name, branches)
+    end
+
+    def delete_branches
+      repository.delete_remote_branches(project.path_with_namespace, mirror.ref_name, deleted_branches)
+    end
+
+    def deleted_branches
+      @deleted_branches ||= remote_branches.each_with_object([]) do |(name, branch), branches|
+        local_branch = local_branches[name]
+
+        if local_branch.nil? && project.commit(branch.target)
+          branches << name
+        end
+      end
+    end
+
+    def changed_branches
+      @changed_branches ||= local_branches.each_with_object([]) do |(name, branch), branches|
+        remote_branch = remote_branches[name]
+
+        if remote_branch.nil?
+          branches << name
+        elsif branch.target == remote_branch.target
+          # Already up to date
+        elsif !repository.upstream_has_diverged?(name, mirror.ref_name)
+          branches << name
+        end
+      end
+    end
+
+    def divergent_branches
+      remote_branches.each_with_object([]) do |(name, _), branches|
+        if local_branches[name] && repository.upstream_has_diverged?(name, mirror.ref_name)
+          branches << name
+        end
+      end
+    end
+
+    def local_tags
+      @local_tags ||= repository.tags.each_with_object({}) do |tag, tags|
+        tags[tag.name] = tag
+      end
+    end
+
+    def remote_tags
+      @remote_tags ||= repository.remote_tags(mirror.ref_name).each_with_object({}) do |tag, tags|
+        tags[tag.name] = tag
+      end
+    end
+
+    def push_tags
+      repository.push_remote_branches(project.path_with_namespace, mirror.ref_name, changed_tags)
+    end
+
+    def delete_tags
+      repository.delete_remote_branches(project.path_with_namespace, mirror.ref_name, deleted_tags)
+    end
+
+    def changed_tags
+      @changed_tags ||= local_tags.each_with_object([]) do |(name, tag), tags|
+        remote_tag = remote_tags[name]
+
+        if remote_tag.nil? || (tag.target != remote_tag.target)
+          tags << name
+        end
+      end
+    end
+
+    def deleted_tags
+      @deleted_tags ||= remote_tags.each_with_object([]) do |(name, _), tags|
+        tags << name if local_tags[name].nil?
+      end
+    end
+
+  end
+end

app/views/projects/buttons/_update_mirror.html.haml

-- if @project.mirror? && can?(current_user, :push_code, @project)
-  - size = nil unless defined?(size) && size
-  - if @project.updating_mirror?
-    %span.btn.disabled.update-mirror-button.has-tooltip{title: "Updating from upstream..."}
-      = icon('refresh')
-  - else
-    = link_to update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn update-mirror-button has-tooltip", title: "Update from upstream" do
-      = icon('refresh')
+- if can?(current_user, :push_code, @project)
+  - if !@project.has_remote_mirror? && @project.mirror?
+    - size = nil unless defined?(size) && size
+    - if @project.updating_mirror?
+      %span.btn.disabled.update-mirror-button.has-tooltip{title: "Updating from upstream..."}
+        = icon('refresh')
+    - else
+      = link_to update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn update-mirror-button has-tooltip", title: "Update from upstream" do
+        = icon('refresh')
+  - elsif @project.has_remote_mirror? && !@project.mirror?
+    - if @project.updating_remote_mirror?
+      %span.btn.disabled.update-mirror-button.has-tooltip{title: "Updating remote repository..."}
+        = icon('refresh')
+    - else
+      = link_to update_now_namespace_project_mirror_path(@project.namespace, @project, sync_remote: true), method: :post, class: "btn update-mirror-button has-tooltip", title: "Update remote repository" do
+        = icon('refresh')
+  - elsif @project.has_remote_mirror? && @project.mirror?
+    .btn-group
+      %a.btn.dropdown-toggle{href: '#', 'data-toggle' => 'dropdown'}
+        = icon('refresh')
+      %ul.dropdown-menu.dropdown-menu-right
+        %li
+          - if @project.updating_mirror?
+            %span.prepend-left-10.disabled-item Updating from upstream...
+          - else
+            = link_to "Update this repository", update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post
+        %li
+          - if @project.updating_remote_mirror?
+            %span.prepend-left-10.disabled-item Updating remote repository...
+          - else
+            = link_to "Update remote repository", update_now_namespace_project_mirror_path(@project.namespace, @project, sync_remote: true), method: :post

app/views/projects/mirrors/show.html.haml

 - page_title "Mirror Repository"
-.pull-right
-  - if @project.mirror_last_update_success?
-    Successfully updated #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
-  %span.prepend-left-10
-    = render "shared/mirror_update_button"
 %h3.page-title
   Mirror Repository
 %p.light
-  Set up your project to automatically have its branches, tags, and commits updated from an upstream repository every hour.
-%hr.clearfix
-- if @project.mirror_last_update_failed?
-  .panel.panel-danger
-    .panel-heading
-      The repository failed to update #{time_ago_with_tooltip(@project.mirror_last_update_at)}.
-      - if @project.mirror_ever_updated_successfully?
-        Last successful update #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
-    .panel-body
-      %pre
-        :preserve
-          #{@project.import_error.try(:strip)}
+  A repository can be setup as a mirror of another repository, and can also have a remote mirror associated.
+%p.light
+  When the repository is configured as a mirror, all of its branches, tags, and commits will automatically be updated from the repository configured in the
+  %strong Pull from a remote repository
+  section.
+%p.light
+  When the repository has a remote mirror associated, it means that the remote repository configured in the <strong>Push to a remote repository</strong> section will automatically receive updates from the current repository.
+
+%hr.clearfix
 = form_for @project, url: namespace_project_mirror_path(@project.namespace, @project), html: { class: 'form-horizontal' } do |f|
   - if @project.errors.any?
@@ -29,6 +21,25 @@
       - @project.errors.full_messages.each do |msg|
         %p= msg
+  %h4 Pull from a remote repository
+  %p.light
+    Set up your project to automatically have its branches, tags, and commits updated from an upstream repository every hour.
+  = render "shared/mirror_update_button"
+  - if @project.mirror_last_update_success?
+    %span.prepend-left-default Successfully updated #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
+  %hr.clearfix
+
+  - if @project.mirror_last_update_failed?
+    .panel.panel-danger
+      .panel-heading
+        The repository failed to update #{time_ago_with_tooltip(@project.mirror_last_update_at)}.
+        - if @project.mirror_ever_updated_successfully?
+          Last successful update #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
+      .panel-body
+        %pre
+          :preserve
+            #{@project.import_error.try(:strip)}
+
   .form-group
     .col-sm-offset-2.col-sm-10
       .checkbox
@@ -69,5 +80,44 @@
   - if @project.builds_enabled?
     = render 'shared/mirror_trigger_builds_setting', f: f
+  %h4 Push to a remote repository
+  %p.light
+    Set up the remote repository that you want to update with the content of the current repository every hour.
+  = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
+  - if @remote_mirror.last_successful_update_at
+    %span.prepend-left-default Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
+
+  %hr.clearfix
+
+  - if @remote_mirror.last_error.present?
+    .panel.panel-danger
+      .panel-heading
+        The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
+        - if @remote_mirror.last_successful_update_at
+          Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
+      .panel-body
+        %pre
+          :preserve
+            #{@remote_mirror.last_error.strip}
+
+  = f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = rm_form.label :enabled do
+            = rm_form.check_box :enabled
+            %strong
+              Remote Mirror Repository
+          .help-block
+            Automatically update the remote mirror's branches, tags, and commits from this repository every hour.
+
+    .form-group.has-feedback
+      = rm_form.label :url, class: 'control-label' do
+        %span Git repository URL
+      .col-sm-10
+        = rm_form.text_field :url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git'
+        .well.prepend-top-20
+          The requirements for the URL format are the same mentioned in the <strong>Pull from a remote repository</strong> section.
+
   .form-actions
     = f.submit "Save Changes", class: "btn btn-create"

app/views/shared/_remote_mirror_update_button.html.haml

+- if @project.has_remote_mirror?
+  - if remote_mirror.update_in_progress?
+    %span.btn.disabled
+      = icon('refresh')
+      Updating…
+  - else
+    = link_to update_now_namespace_project_mirror_path(@project.namespace, @project, sync_remote: true), method: :post, class: "btn" do
+      = icon('refresh')
+      Update Now

app/workers/repository_update_mirror_worker.rb

@@ -12,8 +12,7 @@ class RepositoryUpdateMirrorWorker
     result = Projects::UpdateMirrorService.new(@project, @current_user).execute
     if result[:status] == :error
-      project.update(import_error: result[:message])
-      project.import_fail
+      project.mark_import_as_failed(result[:message])
       return
     end

app/workers/repository_update_remote_mirror_worker.rb

+class RepositoryUpdateRemoteMirrorWorker
+  include Sidekiq::Worker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options queue: :gitlab_shell
+
+  def perform(remote_mirror_id)
+    remote_mirror = RemoteMirror.find(remote_mirror_id)
+    project       = remote_mirror.project
+    current_user  = project.creator
+    result        = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
+
+    if result[:status] == :error
+      remote_mirror.mark_as_failed(result[:message])
+    else
+      remote_mirror.update_finish
+    end
+  end
+end

app/workers/update_all_mirrors_worker.rb

@@ -13,8 +13,7 @@ class UpdateAllMirrorsWorker
       where('mirror_last_update_at < ?', 1.day.ago)
     stuck.find_each(batch_size: 50) do |project|
-      project.import_fail
-      project.update_attribute(:import_error, 'The mirror update took too long to complete.')
+      project.mark_import_as_failed('The mirror update took too long to complete.')
     end
   end
 end

app/workers/update_all_remote_mirrors_worker.rb

+class UpdateAllRemoteMirrorsWorker
+  include Sidekiq::Worker
+
+  def perform
+    fail_stuck_mirrors!
+
+    RemoteMirror.find_each(batch_size: 50).each(&:sync)
+  end
+
+  def fail_stuck_mirrors!
+    RemoteMirror.stuck.find_each(batch_size: 50) do |remote_mirror|
+      remote_mirror.mark_as_failed('The mirror update took too long to complete.')
+    end
+  end
+end

config/gitlab.yml.example

@@ -190,6 +190,10 @@ production: &base
     update_all_mirrors_worker:
       cron: "0 * * * *"
+    # Update remote mirrors
+    update_all_remote_mirrors_worker:
+      cron: "30 * * * *"
+
     # In addition to refreshing users when they log in,
     # periodically refresh LDAP users membership.
     # NOTE: This will only take effect if LDAP is enabled

config/initializers/1_settings.rb

@@ -326,6 +326,9 @@ Settings.cron_jobs['historical_data_worker']['job_class'] = 'HistoricalDataWorke
 Settings.cron_jobs['update_all_mirrors_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['update_all_mirrors_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['update_all_mirrors_worker']['job_class'] = 'UpdateAllMirrorsWorker'
+Settings.cron_jobs['update_all_remote_mirrors_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['update_all_remote_mirrors_worker']['cron'] ||= '30 * * * *'
+Settings.cron_jobs['update_all_remote_mirrors_worker']['job_class'] = 'UpdateAllRemoteMirrorsWorker'
 Settings.cron_jobs['ldap_sync_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['ldap_sync_worker']['cron'] ||= '30 1 * * *'
 Settings.cron_jobs['ldap_sync_worker']['job_class'] = 'LdapSyncWorker'

db/migrate/20160321161032_create_remote_mirrors.rb

+class CreateRemoteMirrors < ActiveRecord::Migration
+  def change
+    create_table :remote_mirrors do |t|
+      t.references :project, index: true, foreign_key: true
+      t.string :url
+      t.boolean :enabled, default: true
+      t.string :update_status
+      t.datetime :last_update_at
+      t.datetime :last_successful_update_at
+      t.string :last_error
+      t.text :encrypted_credentials
+      t.string :encrypted_credentials_iv
+      t.string :encrypted_credentials_salt
+
+      t.timestamps null: false
+    end
+  end
+end

db/schema.rb

@@ -883,6 +883,23 @@ ActiveRecord::Schema.define(version: 20160331223143) do
   add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree
   add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree
+  create_table "remote_mirrors", force: :cascade do |t|
+    t.integer  "project_id"
+    t.string   "url"
+    t.boolean  "enabled",                   default: true
+    t.string   "update_status"
+    t.datetime "last_update_at"
+    t.datetime "last_successful_update_at"
+    t.string   "last_error"
+    t.text     "encrypted_credentials"
+    t.string   "encrypted_credentials_iv"
+    t.string   "encrypted_credentials_salt"
+    t.datetime "created_at",                                null: false
+    t.datetime "updated_at",                                null: false
+  end
+
+  add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
+
   create_table "sent_notifications", force: :cascade do |t|
     t.integer "project_id"
     t.integer "noteable_id"

lib/gitlab/backend/shell.rb

@@ -41,6 +41,31 @@ module Gitlab
       true
     end
+    def list_remote_tags(name, remote)
+      output, status = Popen::popen([gitlab_shell_projects_path, 'list-remote-tags', "#{name}.git", remote])
+      tags_with_targets = []
+
+      raise Error, output unless status.zero?
+
+      # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
+      # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]
+      output.lines.each do |line|
+        target, path = line.strip!.split("\t")
+
+        # When the remote repo is empty we don't have tags.
+        break if target.nil?
+
+        name = path.split('/', 3).last
+        # We're only interested in tag references
+        # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name
+        next if name =~ /\^\{\}\Z/
+
+        tags_with_targets.concat([name, target])
+      end
+
+      Hash[*tags_with_targets]
+    end
+
     # Fetch remote for repository
     #
     # name - project path with namespace
@@ -50,9 +75,11 @@ module Gitlab
     # Ex.
     #   fetch_remote("gitlab/gitlab-ci", "upstream")
     #
-    def fetch_remote(name, remote, forced: false)
+    def fetch_remote(name, remote, forced: false, no_tags: false)
       args = [gitlab_shell_projects_path, 'fetch-remote', "#{name}.git", remote, '600']
       args << '--force' if forced
+      args << '--no-tags' if no_tags
+
       output, status = Popen::popen(args)
       raise Error, output unless status.zero?
       true
@@ -271,6 +298,38 @@ module Gitlab
       File.exists?(full_path(dir_name))
     end
+    # Push branch to remote repository
+    #
+    # project_name - project's name with namespace
+    # remote_name - remote name
+    # branch_name - remote branch name
+    #
+    # Ex.
+    #   push_remote_branches('upstream', 'feature')
+    #
+    def push_remote_branches(project_name, remote_name, branch_names)
+      args = [gitlab_shell_projects_path, 'push-branches', "#{project_name}.git", remote_name, *branch_names]
+      output, status = Popen::popen(args)
+      raise Error, output unless status.zero?
+      true
+    end
+
+    # Delete branch from remote repository
+    #
+    # project_name - project's name with namespace
+    # remote_name - remote name
+    # branch_name - remote branch name
+    #
+    # Ex.
+    #   delete_remote_branches('upstream', 'feature')
+    #
+    def delete_remote_branches(project_name, remote_name, branch_names)
+      args = [gitlab_shell_projects_path, 'delete-remote-branches', "#{project_name}.git", remote_name, *branch_names]
+      output, status = Popen::popen(args)
+      raise Error, output unless status.zero?
+      true
+    end
+
     protected
     def gitlab_shell_path