Merge branch 'gitlab_importer' into 'master'

Gitlab.com integration && Gitlab importer

#1933

See merge request !1456

CHANGELOG

@@ -32,6 +32,8 @@ v 7.8.0
   - Disable blacklist validation for project names
   - Allow configuring protection of the default branch upon first push (Marco Wessel)
   -
+  - Add gitlab.com importer
+  - Add an ability to login with gitlab.com
   - 
   - Add a commit calendar to the user profile (Hannes Rosenögger)  
   - 

Gemfile

@@ -29,6 +29,7 @@ gem 'omniauth-twitter'
 gem 'omniauth-github'
 gem 'omniauth-shibboleth'
 gem 'omniauth-kerberos'
+gem 'omniauth-gitlab'
 gem 'doorkeeper', '2.1.0'
 gem "rack-oauth2", "~> 1.0.5"

Gemfile.lock

@@ -332,6 +332,9 @@ GEM
     omniauth-github (1.1.1)
       omniauth (~> 1.0)
       omniauth-oauth2 (~> 1.1)
+    omniauth-gitlab (1.0.0)
+      omniauth (~> 1.0)
+      omniauth-oauth2 (~> 1.0)
     omniauth-google-oauth2 (0.2.5)
       omniauth (> 1.0)
       omniauth-oauth2 (~> 1.1)
@@ -689,6 +692,7 @@ DEPENDENCIES
   octokit (= 3.7.0)
   omniauth (~> 1.1.3)
   omniauth-github
+  omniauth-gitlab
   omniauth-google-oauth2
   omniauth-kerberos
   omniauth-shibboleth

app/assets/javascripts/importer_status.js.coffee

+class @ImporterStatus
+  constructor: (@jobs_url, @import_url) ->
+    this.initStatusPage()
+    this.setAutoUpdate()
+
+  initStatusPage: ->
+    $(".btn-add-to-import").click (event) =>
+      new_namespace = null
+      tr = $(event.currentTarget).closest("tr")
+      id = tr.attr("id").replace("repo_", "")
+      if tr.find(".import-target input").length > 0
+        new_namespace = tr.find(".import-target input").prop("value")
+        tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
+      $.post @import_url, {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
+  
+  setAutoUpdate: ->
+    setInterval (=>
+      $.get @jobs_url, (data) =>
+        $.each data, (i, job) =>
+          job_item = $("#project_" + job.id)
+          status_field = job_item.find(".job-status")
+          
+          if job.import_status == 'finished'
+            job_item.removeClass("active").addClass("success")
+            status_field.html('<span class="cgreen"><i class="fa fa-check"></i> done</span>')
+          else if job.import_status == 'started'
+            status_field.html("<i class='fa fa-spinner fa-spin'></i> started")
+          else
+            status_field.html(job.import_status)
+          
+    ), 4000
\ No newline at end of file

app/controllers/import/base_controller.rb

+class Import::BaseController < ApplicationController
+
+  private
+
+  def get_or_create_namespace
+    existing_namespace = Namespace.find_by("path = ? OR name = ?", @target_namespace, @target_namespace)
+
+    if existing_namespace
+      if existing_namespace.owner == current_user
+        namespace = existing_namespace
+      else
+        @already_been_taken = true
+        return false
+      end
+    else
+      namespace = Group.create(name: @target_namespace, path: @target_namespace, owner: current_user)
+      namespace.add_owner(current_user)
+      namespace
+    end
+  end
+end

app/controllers/github_imports_controller.rb → app/controllers/import/github_controller.rb

-class GithubImportsController < ApplicationController
+class Import::GithubController < Import::BaseController
   before_filter :github_auth, except: :callback
   rescue_from Octokit::Unauthorized, with: :github_unauthorized
@@ -7,7 +7,7 @@ class GithubImportsController < ApplicationController
     token = client.auth_code.get_token(params[:code]).token
     current_user.github_access_token = token
     current_user.save
-    redirect_to status_github_import_url
+    redirect_to status_import_github_url
   end
   def status
@@ -30,30 +30,18 @@ class GithubImportsController < ApplicationController
   def create
     @repo_id = params[:repo_id].to_i
     repo = octo_client.repo(@repo_id)
-    target_namespace = params[:new_namespace].presence || repo.owner.login
-    existing_namespace = Namespace.find_by("path = ? OR name = ?", target_namespace, target_namespace)
-
-    if existing_namespace
-      if existing_namespace.owner == current_user
-        namespace = existing_namespace
-      else
-        @already_been_taken = true
-        @target_namespace = target_namespace
+    @target_namespace = params[:new_namespace].presence || repo.owner.login
     @project_name = repo.name
-        render and return
-      end
-    else
-      namespace = Group.create(name: target_namespace, path: target_namespace, owner: current_user)
-      namespace.add_owner(current_user)
-    end
     
-    @project = Gitlab::Github::ProjectCreator.new(repo, namespace, current_user).execute
+    namespace = get_or_create_namespace || (render and return)
+
+    @project = Gitlab::GithubImport::ProjectCreator.new(repo, namespace, current_user).execute
   end
   private
   def client
-    @client ||= Gitlab::Github::Client.new.client
+    @client ||= Gitlab::GithubImport::Client.new.client
   end
   def octo_client
@@ -69,7 +57,7 @@ class GithubImportsController < ApplicationController
   def go_to_github_for_permissions
     redirect_to client.auth_code.authorize_url({
-      redirect_uri: callback_github_import_url,
+      redirect_uri: callback_import_github_url,
       scope: "repo, user, user:email"
     })
   end

app/controllers/import/gitlab_controller.rb

+class Import::GitlabController < Import::BaseController
+  before_filter :gitlab_auth, except: :callback
+
+  rescue_from OAuth2::Error, with: :gitlab_unauthorized
+
+  def callback
+    token = client.get_token(params[:code], callback_import_gitlab_url)
+    current_user.gitlab_access_token = token
+    current_user.save
+    redirect_to status_import_gitlab_url
+  end
+
+  def status
+    @repos = client.projects
+    
+    @already_added_projects = current_user.created_projects.where(import_type: "gitlab")
+    already_added_projects_names = @already_added_projects.pluck(:import_source)
+
+    @repos.to_a.reject!{ |repo| already_added_projects_names.include? repo["path_with_namespace"] }
+  end
+
+  def jobs
+    jobs = current_user.created_projects.where(import_type: "gitlab").to_json(only: [:id, :import_status])
+    render json: jobs
+  end
+
+  def create
+    @repo_id = params[:repo_id].to_i
+    repo = client.project(@repo_id)
+    @target_namespace = params[:new_namespace].presence || repo["namespace"]["path"]
+    @project_name = repo["name"]
+    
+    namespace = get_or_create_namespace || (render and return)
+
+    @project = Gitlab::GitlabImport::ProjectCreator.new(repo, namespace, current_user).execute
+  end
+
+  private
+
+  def client
+    @client ||= Gitlab::GitlabImport::Client.new(current_user.gitlab_access_token)
+  end
+
+  def gitlab_auth
+    if current_user.gitlab_access_token.blank?
+      go_to_gitlab_for_permissions
+    end
+  end
+
+  def go_to_gitlab_for_permissions
+    redirect_to client.authorize_url(callback_import_gitlab_url)
+  end
+
+  def gitlab_unauthorized
+    go_to_gitlab_for_permissions
+  end
+end

app/helpers/oauth_helper.rb

@@ -4,7 +4,7 @@ module OauthHelper
   end
   def default_providers
-    [:twitter, :github, :google_oauth2, :ldap]
+    [:twitter, :github, :gitlab, :google_oauth2, :ldap]
   end
   def enabled_oauth_providers
@@ -13,7 +13,7 @@ module OauthHelper
   def enabled_social_providers
     enabled_oauth_providers.select do |name|
-      [:twitter, :github, :google_oauth2].include?(name.to_sym)
+      [:twitter, :gitlab, :github, :google_oauth2].include?(name.to_sym)
     end
   end

app/helpers/projects_helper.rb

@@ -253,4 +253,8 @@ module ProjectsHelper
   def github_import_enabled?
     enabled_oauth_providers.include?(:github)
   end
+
+  def gitlab_import_enabled?
+    enabled_oauth_providers.include?(:gitlab)
+  end
 end

app/views/github_imports/status.html.haml → app/views/import/github/status.html.haml

@@ -17,7 +17,7 @@
       %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
         %td= project.import_source
         %td
-          %strong= link_to project.name_with_namespace, project
+          %strong= link_to project.path_with_namespace, project
         %td.job-status
           - if project.import_status == 'finished'
             %span.cgreen
@@ -34,30 +34,6 @@
         %td.import-actions.job-status
           = button_tag "Add", class: "btn btn-add-to-import"
-
 :coffeescript
-  $(".btn-add-to-import").click () ->
-    new_namespace = null
-    tr = $(this).closest("tr")
-    id = tr.attr("id").replace("repo_", "")
-    if tr.find(".import-target input").length > 0
-      new_namespace = tr.find(".import-target input").prop("value")
-      tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
-    $.post "#{github_import_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
-  
-
-  setInterval (->
-    $.get "#{jobs_github_import_path}", (data)->
-      $.each data, (i, job) ->
-        job_item = $("#project_" + job.id)
-        status_field = job_item.find(".job-status")
-        
-        if job.import_status == 'finished'
-          job_item.removeClass("active").addClass("success")
-          status_field.html('<span class="cgreen"><i class="fa fa-check"></i> done</span>')
-        else if job.import_status == 'started'
-          status_field.html("<i class='fa fa-spinner fa-spin'></i> started")
-        else
-          status_field.html(job.import_status)
-        
-  ), 4000
+  $ ->
+    new ImporterStatus("#{jobs_import_github_path}", "#{import_github_path}")

app/views/import/gitlab/status.html.haml

+%h3.page-title
+  %i.fa.fa-github
+  Import repositories from GitLab.com
+
+%p.light
+  Select projects you want to import.  
+
+%hr
+%table.table.import-jobs
+  %thead
+    %tr
+      %th From GitLab.com
+      %th To GitLab private instance
+      %th Status
+  %tbody
+    - @already_added_projects.each do |project|
+      %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
+        %td= project.import_source
+        %td
+          %strong= link_to project.path_with_namespace, project
+        %td.job-status
+          - if project.import_status == 'finished'
+            %span.cgreen
+              %i.fa.fa-check
+              done
+          - else
+            = project.human_import_status_name
+
+    - @repos.each do |repo|
+      %tr{id: "repo_#{repo["id"]}"}
+        %td= repo["path_with_namespace"]
+        %td.import-target
+          = repo["path_with_namespace"]
+        %td.import-actions.job-status
+          = button_tag "Add", class: "btn btn-add-to-import"
+
+:coffeescript
+  $ -> 
+    new ImporterStatus("#{jobs_import_gitlab_path}", "#{import_gitlab_url}")

app/views/projects/_github_import_modal.html.haml

@@ -7,16 +7,3 @@
       .modal-body
         You need to setup integration with GitHub first.
         = link_to 'How to setup integration with GitHub', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/github.md'
-
-
-:javascript
-  $(function(){
-    var import_modal = $('#github_import_modal').modal({modal: true, show:false});
-    $('.how_to_import_link').bind("click", function(e){
-      e.preventDefault();
-      import_modal.show();
-    });
-    $('.modal-header .close').bind("click", function(){
-      import_modal.hide();
-    })
-  })

app/views/projects/_gitlab_import_modal.html.haml

+%div#gitlab_import_modal.modal.hide
+  .modal-dialog
+    .modal-content
+      .modal-header
+        %a.close{href: "#", "data-dismiss" => "modal"} ×
+        %h3 GitLab OAuth import
+      .modal-body
+        You need to setup integration with GitLab first.
+        = link_to 'How to setup integration with GitLab', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/gitlab.md'
\ No newline at end of file

app/views/projects/new.html.haml

@@ -44,7 +44,7 @@
         .col-sm-2
         .col-sm-10
           - if github_import_enabled?
-            = link_to status_github_import_path do
+            = link_to status_import_github_path do
               %i.fa.fa-github
               Import projects from GitHub
           - else
@@ -53,6 +53,19 @@
               Import projects from GitHub
             = render 'github_import_modal'
       
+      .project-import.form-group
+        .col-sm-2
+        .col-sm-10
+          - if gitlab_import_enabled?
+            = link_to status_import_gitlab_path do
+              %i.fa.fa-heart
+              Import projects from GitLab.com
+          - else
+            = link_to '#', class: 'how_to_import_link light' do
+              %i.fa.fa-heart
+              Import projects from GitLab.com
+            = render 'gitlab_import_modal'
+
       %hr.prepend-botton-10
       .form-group
@@ -79,3 +92,11 @@
       %i.fa.fa-spinner.fa-spin
       Creating project & repository.
     %p Please wait a moment, this page will automatically refresh when ready.
+
+:coffeescript
+  $ ->
+    $('.how_to_import_link').bind 'click', (e) ->
+      e.preventDefault()
+      import_modal = $(this).parent().find(".modal").show()
+    $('.modal-header .close').bind 'click', ->
+      $(".modal").hide()
\ No newline at end of file

app/workers/repository_import_worker.rb

@@ -10,10 +10,12 @@ class RepositoryImportWorker
                                project.path_with_namespace,
                                project.import_url)
-    if project.import_type == 'github'
-      result_of_data_import = Gitlab::Github::Importer.new(project).execute
+    result_of_data_import = if project.import_type == 'github'
+                              Gitlab::GithubImport::Importer.new(project).execute
+                            elsif project.import_type == 'gitlab'
+                              Gitlab::GitlabImport::Importer.new(project).execute
                             else
-      result_of_data_import = true
+                              true
                             end
     if result && result_of_data_import

config/initializers/doorkeeper.rb

@@ -27,7 +27,7 @@ Doorkeeper.configure do
   # Access token expiration time (default 2 hours).
   # If you want to disable expiration, set this to nil.
-  # access_token_expires_in 2.hours
+  access_token_expires_in nil
   # Reuse access token for the same resource owner within an application (disabled by default)
   # Rationale: https://github.com/doorkeeper-gem/doorkeeper/issues/383

config/routes.rb

@@ -51,15 +51,26 @@ Gitlab::Application.routes.draw do
   end
   get '/s/:username' => 'snippets#user_index', as: :user_snippets, constraints: { username: /.*/ }
+  
   #
-  # Github importer area
+  # Import
   #
-  resource :github_import, only: [:create, :new] do
+  namespace :import do
+    resource :github, only: [:create, :new], controller: :github do
       get :status
       get :callback
       get :jobs
     end
+    resource :gitlab, only: [:create, :new], controller: :gitlab do
+      get :status
+      get :callback
+      get :jobs
+    end
+  end
+  
+  
+
   #
   # Explore area
   #