Merge branch 'bvl-phabricator-import-issues' into 'master'

Setup Phabricator import

Closes #60562

See merge request gitlab-org/gitlab-ce!27877

app/controllers/application_controller.rb

@@ -42,7 +42,7 @@ class ApplicationController < ActionController::Base
     :bitbucket_server_import_enabled?,
     :google_code_import_enabled?, :fogbugz_import_enabled?,
     :git_import_enabled?, :gitlab_project_import_enabled?,
-    :manifest_import_enabled?
+    :manifest_import_enabled?, :phabricator_import_enabled?
   # Adds `no-store` to the DEFAULT_CACHE_CONTROL, to prevent security
   # concerns due to caching private data.
@@ -424,6 +424,10 @@ class ApplicationController < ActionController::Base
     Group.supports_nested_objects? && Gitlab::CurrentSettings.import_sources.include?('manifest')
   end
+  def phabricator_import_enabled?
+    Gitlab::PhabricatorImport.available?
+  end
+
   # U2F (universal 2nd factor) devices need a unique identifier for the application
   # to perform authentication.
   # https://developers.yubico.com/U2F/App_ID.html

app/controllers/import/phabricator_controller.rb

+# frozen_string_literal: true
+
+class Import::PhabricatorController < Import::BaseController
+  include ImportHelper
+
+  before_action :verify_import_enabled
+
+  def new
+  end
+
+  def create
+    @project = Gitlab::PhabricatorImport::ProjectCreator
+                 .new(current_user, import_params).execute
+
+    if @project&.persisted?
+      redirect_to @project
+    else
+      @name = params[:name]
+      @path = params[:path]
+      @errors = @project&.errors&.full_messages || [_("Invalid import params")]
+
+      render :new
+    end
+  end
+
+  def verify_import_enabled
+    render_404 unless phabricator_import_enabled?
+  end
+
+  private
+
+  def import_params
+    params.permit(:path, :phabricator_server_url, :api_token, :name, :namespace_id)
+  end
+end

app/views/import/gitlab_projects/new.html.haml

@@ -7,28 +7,7 @@
 %hr
 = form_tag import_gitlab_project_path, class: 'new_project', multipart: true do
-  .row
-    .form-group.project-name.col-sm-12
-      = label_tag :name, _('Project name'), class: 'label-bold'
-      = text_field_tag :name, @name, placeholder: "My awesome project", class: "js-project-name form-control input-lg", autofocus: true
-    .form-group.col-12.col-sm-6
-      = label_tag :namespace_id, _('Project URL'), class: 'label-bold'
-      .form-group
-        .input-group
-          - if current_user.can_select_namespace?
-            .input-group-prepend.has-tooltip{ title: root_url }
-              .input-group-text
-                = root_url
-            = select_tag :namespace_id, namespaces_options(namespace_id_from(params) || :current_user, display_path: true, extra_group: namespace_id_from(params)), class: 'select2 js-select-namespace', tabindex: 1
-
-          - else
-            .input-group-prepend.static-namespace.has-tooltip{ title: user_url(current_user.username) + '/' }
-              .input-group-text.border-0
-                #{user_url(current_user.username)}/
-            = hidden_field_tag :namespace_id, value: current_user.namespace_id
-    .form-group.col-12.col-sm-6.project-path
-      = label_tag :path, _('Project slug'), class: 'label-bold'
-      = text_field_tag :path, @path, placeholder: "my-awesome-project", class: "js-path-name form-control", tabindex: 2, required: true
+  = render 'import/shared/new_project_form'
   .row
     .form-group.col-md-12

app/views/import/manifest/new.html.haml

@@ -4,9 +4,5 @@
 %h3.page-title
   = _('Manifest file import')
-- if @errors.present?
-  .alert.alert-danger
-    - @errors.each do |error|
-      = error
-
+= render 'import/shared/errors'
 = render 'form'

app/views/import/phabricator/new.html.haml

+- title = _('Phabricator Server Import')
+- page_title title
+- breadcrumb_title title
+- header_title _("Projects"), root_path
+
+%h3.page-title
+  = icon 'issues', text: _('Import tasks from Phabricator into issues')
+
+= render 'import/shared/errors'
+
+= form_tag import_phabricator_path, class: 'new_project', method: :post do
+  = render 'import/shared/new_project_form'
+
+  %h4.prepend-top-0= _('Enter in your Phabricator Server URL and personal access token below')
+
+  .form-group.row
+    = label_tag :phabricator_server_url, _('Phabricator Server URL'), class: 'col-form-label col-md-2'
+    .col-md-4
+      = text_field_tag :phabricator_server_url, params[:phabricator_server_url], class: 'form-control append-right-8', placeholder: 'https://your-phabricator-server', size: 40
+  .form-group.row
+    = label_tag :api_token, _('API Token'), class: 'col-form-label col-md-2'
+    .col-md-4
+      = password_field_tag :api_token, params[:api_token], class: 'form-control append-right-8', placeholder: _('Personal Access Token'), size: 40
+  .form-actions
+    = submit_tag _('Import tasks'), class: 'btn btn-success'

app/views/import/shared/_errors.html.haml

+- if @errors.present?
+  .alert.alert-danger
+    - @errors.each do |error|
+      = error

app/views/import/shared/_new_project_form.html.haml

+.row
+  .form-group.project-name.col-sm-12
+    = label_tag :name, _('Project name'), class: 'label-bold'
+    = text_field_tag :name, @name, placeholder: "My awesome project", class: "js-project-name form-control input-lg", autofocus: true
+  .form-group.col-12.col-sm-6
+    = label_tag :namespace_id, _('Project URL'), class: 'label-bold'
+    .form-group
+      .input-group.flex-nowrap
+        - if current_user.can_select_namespace?
+          .input-group-prepend.flex-shrink-0.has-tooltip{ title: root_url }
+            .input-group-text
+              = root_url
+          = select_tag :namespace_id, namespaces_options(namespace_id_from(params) || :current_user, display_path: true, extra_group: namespace_id_from(params)), class: 'select2 js-select-namespace', tabindex: 1
+        - else
+          .input-group-prepend.static-namespace.has-tooltip{ title: user_url(current_user.username) + '/' }
+            .input-group-text.border-0
+              #{user_url(current_user.username)}/
+          = hidden_field_tag :namespace_id, value: current_user.namespace_id
+  .form-group.col-12.col-sm-6.project-path
+    = label_tag :path, _('Project slug'), class: 'label-bold'
+    = text_field_tag :path, @path, placeholder: "my-awesome-project", class: "js-path-name form-control", tabindex: 2, required: true

app/views/projects/_import_project_pane.html.haml

@@ -63,6 +63,13 @@
           = link_to new_import_manifest_path, class: 'btn import_manifest', data: { track_label: "#{track_label}", track_event: "click_button", track_property: "manifest_file" } do
             = icon('file-text-o', text: 'Manifest file')
+      - if phabricator_import_enabled?
+        %div
+          = link_to new_import_phabricator_path, class: 'btn import_phabricator', data: { track_label: "#{track_label}", track_event: "click_button", track_property: "phabricator" } do
+            = custom_icon('issues')
+            = _("Phabricator Tasks")
+
+
   .js-toggle-content.toggle-import-form{ class: ('hide' if active_tab != 'import') }
     = form_for @project, html: { class: 'new_project' } do |f|
       %hr

config/routes/import.rb

@@ -67,4 +67,6 @@ namespace :import do
     get :jobs
     post :upload
   end
+
+  resource :phabricator, only: [:create, :new], controller: :phabricator
 end

config/sidekiq_queues.yml

@@ -91,3 +91,4 @@
   - [chat_notification, 2]
   - [migrate_external_diffs, 1]
   - [update_project_statistics, 1]
+  - [phabricator_import_import_tasks, 1]

doc/user/project/import/index.md

@@ -14,6 +14,7 @@
 1. [From repo by URL](repo_by_url.md)
 1. [By uploading a manifest file (AOSP)](manifest.md)
 1. [From Gemnasium](gemnasium.md)
+1. [From Phabricator](phabricator.md)
 In addition to the specific migration documentation above, you can import any
 Git repository via HTTP from the New Project page. Be aware that if the

doc/user/project/import/phabricator.md

+# Import Phabricator tasks into a GitLab project
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/60562) in
+GitLab 12.0.
+
+GitLab allows you to import all tasks from a Phabricator instance into
+GitLab issues. The import creates a single project with the
+repository disabled.
+
+Currently, only the following basic fields are imported:
+
+- Title
+- Description
+- State (open or closed)
+- Created at
+- Closed at
+
+
+## Enabling this feature
+
+While this feature is incomplete, a feature flag is required to enable it so that
+we can gain early feedback before releasing it for everyone. To enable it:
+
+1. Enable Phabricator as an [import source](../../admin_area/settings/visibility_and_access_controls.md#import-sources) in the Admin area.
+
+  ``` {.ruby}
+  Feature.enable(:phabricator_import)
+  ```
+
+The [import
+source](../../admin_area/settings/visibility_and_access_controls.md#import-sources)
+also needs to be activated by an admin in the admin interface.

lib/gitlab/github_import/parallel_importer.rb

@@ -29,29 +29,13 @@ module Gitlab
       end
       def execute
-        jid = generate_jid
-
-        # The original import JID is the JID of the RepositoryImportWorker job,
-        # which will be removed once that job completes. Reusing that JID could
-        # result in StuckImportJobsWorker marking the job as stuck before we get
-        # to running Stage::ImportRepositoryWorker.
-        #
-        # We work around this by setting the JID to a custom generated one, then
-        # refreshing it in the various stages whenever necessary.
-        Gitlab::SidekiqStatus
-          .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
-
-        project.import_state.update_column(:jid, jid)
+        Gitlab::Import::SetAsyncJid.set_jid(project)
         Stage::ImportRepositoryWorker
           .perform_async(project.id)
         true
       end
-
-      def generate_jid
-        "github-importer/#{project.id}"
-      end
     end
   end
 end

lib/gitlab/http.rb

@@ -9,6 +9,13 @@ module Gitlab
     BlockedUrlError = Class.new(StandardError)
     RedirectionTooDeep = Class.new(StandardError)
+    HTTP_ERRORS = [
+      SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET,
+      Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout,
+      Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError,
+      Gitlab::HTTP::RedirectionTooDeep
+    ].freeze
+
     include HTTParty # rubocop:disable Gitlab/HTTParty
     connection_adapter ProxyHTTPConnectionAdapter

lib/gitlab/import/set_async_jid.rb

+# frozen_string_literal: true
+
+# The original import JID is the JID of the RepositoryImportWorker job,
+# which will be removed once that job completes. Reusing that JID could
+# result in StuckImportJobsWorker marking the job as stuck before we get
+# to running Stage::ImportRepositoryWorker.
+#
+# We work around this by setting the JID to a custom generated one, then
+# refreshing it in the various stages whenever necessary.
+module Gitlab
+  module Import
+    module SetAsyncJid
+      def self.set_jid(project)
+        jid = generate_jid(project)
+
+        Gitlab::SidekiqStatus
+          .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
+
+        project.import_state.update_column(:jid, jid)
+      end
+
+      def self.generate_jid(project)
+        "async-import/#{project.id}"
+      end
+    end
+  end
+end

lib/gitlab/import_sources.rb

@@ -20,7 +20,8 @@ module Gitlab
       ImportSource.new('git',              'Repo by URL',      nil),
       ImportSource.new('gitlab_project',   'GitLab export',    Gitlab::ImportExport::Importer),
       ImportSource.new('gitea',            'Gitea',            Gitlab::LegacyGithubImport::Importer),
-      ImportSource.new('manifest',         'Manifest file',    nil)
+      ImportSource.new('manifest',         'Manifest file',    nil),
+      ImportSource.new('phabricator',      'Phabricator',      Gitlab::PhabricatorImport::Importer)
     ].freeze
     class << self

lib/gitlab/phabricator_import.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module PhabricatorImport
+    BaseError = Class.new(StandardError)
+
+    def self.available?
+      Feature.enabled?(:phabricator_import) &&
+        Gitlab::CurrentSettings.import_sources.include?('phabricator')
+    end
+  end
+end

lib/gitlab/phabricator_import/base_worker.rb

+# frozen_string_literal: true
+
+# All workers within a Phabricator import should inherit from this worker and
+# implement the `#import` method. The jobs should then be scheduled using the
+# `.schedule` class method instead of `.perform_async`
+#
+# Doing this makes sure that only one job of that type is running at the same time
+# for a certain project. This will avoid deadlocks. When a job is already running
+# we'll wait for it for 10 times 5 seconds to restart. If the running job hasn't
+# finished, by then, we'll retry in 30 seconds.
+#
+# It also makes sure that we keep the import state of the project up to date:
+# - It keeps track of the jobs so we know how many jobs are running for the
+#   project
+# - It refreshes the import jid, so it doesn't get cleaned up by the
+#   `StuckImportJobsWorker`
+# - It marks the import as failed if a job failed to many times
+# - It marks the import as finished when all remaining jobs are done
+module Gitlab
+  module PhabricatorImport
+    class BaseWorker
+      include ApplicationWorker
+      include ProjectImportOptions # This marks the project as failed after too many tries
+      include Gitlab::ExclusiveLeaseHelpers
+
+      class << self
+        def schedule(project_id, *args)
+          perform_async(project_id, *args)
+          add_job(project_id)
+        end
+
+        def add_job(project_id)
+          worker_state(project_id).add_job
+        end
+
+        def remove_job(project_id)
+          worker_state(project_id).remove_job
+        end
+
+        def worker_state(project_id)
+          Gitlab::PhabricatorImport::WorkerState.new(project_id)
+        end
+      end
+
+      def perform(project_id, *args)
+        in_lock("#{self.class.name.underscore}/#{project_id}/#{args}", ttl: 2.hours, sleep_sec: 5.seconds) do
+          project = Project.find_by_id(project_id)
+          next unless project
+
+          # Bail if the import job already failed
+          next unless project.import_state&.in_progress?
+
+          project.import_state.refresh_jid_expiration
+
+          import(project, *args)
+
+          # If this is the last running job, finish the import
+          project.after_import if self.class.worker_state(project_id).running_count < 2
+
+          self.class.remove_job(project_id)
+        end
+      rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError
+        # Reschedule a job if there was already a running one
+        # Running them at the same time could cause a deadlock updating the same
+        # resource
+        self.class.perform_in(30.seconds, project_id, *args)
+      end
+
+      private
+
+      def import(project, *args)
+        importer_class.new(project, *args).execute
+      end
+
+      def importer_class
+        raise NotImplementedError, "Implement `#{__method__}` on #{self.class}"
+      end
+    end
+  end
+end

lib/gitlab/phabricator_import/cache/map.rb

+# frozen_string_literal: true
+module Gitlab
+  module PhabricatorImport
+    module Cache
+      class Map
+        def initialize(project)
+          @project = project
+        end
+
+        def get_gitlab_model(phabricator_id)
+          cached_info = get(phabricator_id)
+          return unless cached_info[:classname] && cached_info[:database_id]
+
+          cached_info[:classname].constantize.find_by_id(cached_info[:database_id])
+        end
+
+        def set_gitlab_model(object, phabricator_id)
+          set(object.class, object.id, phabricator_id)
+        end
+
+        private
+
+        attr_reader :project
+
+        def set(klass_name, object_id, phabricator_id)
+          key = cache_key_for_phabricator_id(phabricator_id)
+
+          redis.with do |r|
+            r.multi do |multi|
+              multi.mapped_hmset(key,
+                             { classname: klass_name, database_id: object_id })
+              multi.expire(key, timeout)
+            end
+          end
+        end
+
+        def get(phabricator_id)
+          key = cache_key_for_phabricator_id(phabricator_id)
+
+          redis.with do |r|
+            r.pipelined do |pipe|
+              # Extend the TTL when a key was
+              pipe.expire(key, timeout)
+              pipe.mapped_hmget(key, :classname, :database_id)
+            end.last
+          end
+        end
+
+        def cache_key_for_phabricator_id(phabricator_id)
+          "#{Redis::Cache::CACHE_NAMESPACE}/phabricator-import/#{project.id}/#{phabricator_id}"
+        end
+
+        def redis
+          Gitlab::Redis::Cache
+        end
+
+        def timeout
+          # Setting the timeout to the same one as we do for clearing stuck jobs
+          # this makes sure all cache is available while the import is running.
+          StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION
+        end
+      end
+    end
+  end
+end

lib/gitlab/phabricator_import/conduit.rb

+# frozen_string_literal: true
+module Gitlab
+  module PhabricatorImport
+    module Conduit
+      ApiError = Class.new(Gitlab::PhabricatorImport::BaseError)
+      ResponseError = Class.new(ApiError)
+    end
+  end
+end