diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 67988ea34604892d3169e70215e4fc6ea2fa3219..d325eca6d99cd4ab6a6d0a967a8a0b3a1b305803 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -1,6 +1,9 @@
module Gitlab
module GithubImport
class Client
+ GITHUB_SAFE_REMAINING_REQUESTS = 100
+ GITHUB_SAFE_SLEEP_TIME = 500
+
attr_reader :client, :api
def initialize(access_token)
@@ -11,7 +14,7 @@ module Gitlab
)
if access_token
- ::Octokit.auto_paginate = true
+ ::Octokit.auto_paginate = false
@api = ::Octokit::Client.new(
access_token: access_token,
@@ -36,7 +39,7 @@ module Gitlab
def method_missing(method, *args, &block)
if api.respond_to?(method)
- api.send(method, *args, &block)
+ request { api.send(method, *args, &block) }
else
super(method, *args, &block)
end
@@ -55,6 +58,34 @@ module Gitlab
def github_options
config["args"]["client_options"].deep_symbolize_keys
end
+
+ def rate_limit
+ api.rate_limit!
+ end
+
+ def rate_limit_exceed?
+ rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
+ end
+
+ def rate_limit_sleep_time
+ rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
+ end
+
+ def request
+ sleep rate_limit_sleep_time if rate_limit_exceed?
+
+ data = yield
+
+ last_response = api.last_response
+
+ while last_response.rels[:next]
+ sleep rate_limit_sleep_time if rate_limit_exceed?
+ last_response = last_response.rels[:next].get
+ data.concat(last_response.data) if last_response.data.is_a?(Array)
+ end
+
+ data
+ end
end
end
end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 5ef9d66ba68ebb8421acd859d4dbb758083d681a..e5cf66a037102f70034b56ef765030722921ecf9 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -3,9 +3,6 @@ module Gitlab
class Importer
include Gitlab::ShellAdapter
- GITHUB_SAFE_REMAINING_REQUESTS = 100
- GITHUB_SAFE_SLEEP_TIME = 500
-
attr_reader :client, :project, :repo, :repo_url
def initialize(project)
@@ -28,52 +25,12 @@ module Gitlab
private
- def turn_auto_pagination_off!
- client.auto_paginate = false
- end
-
- def turn_auto_pagination_on!
- client.auto_paginate = true
- end
-
- def rate_limit
- client.rate_limit!
- end
-
- def rate_limit_exceed?
- rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
- end
-
- def rate_limit_sleep_time
- rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
- end
-
- def paginate
- turn_auto_pagination_off!
-
- sleep rate_limit_sleep_time if rate_limit_exceed?
-
- data = yield
-
- last_response = client.last_response
-
- while last_response.rels[:next]
- sleep rate_limit_sleep_time if rate_limit_exceed?
- last_response = last_response.rels[:next].get
- data.concat(last_response.data) if last_response.data.is_a?(Array)
- end
-
- turn_auto_pagination_on!
-
- data
- end
-
def credentials
@credentials ||= project.import_data.credentials if project.import_data
end
def import_labels
- labels = paginate { client.labels(repo, per_page: 100) }
+ labels = client.labels(repo, per_page: 100)
labels.each { |raw| LabelFormatter.new(project, raw).create! }
true
@@ -82,7 +39,7 @@ module Gitlab
end
def import_milestones
- milestones = paginate { client.milestones(repo, state: :all, per_page: 100) }
+ milestones = client.milestones(repo, state: :all, per_page: 100)
milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
true
@@ -91,9 +48,9 @@ module Gitlab
end
def import_issues
- data = paginate { client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
+ issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
- data.each do |raw|
+ issues.each do |raw|
gh_issue = IssueFormatter.new(project, raw)
if gh_issue.valid?
@@ -112,7 +69,7 @@ module Gitlab
hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
disable_webhooks(hooks)
- pull_requests = paginate { client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
+ pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
@@ -146,14 +103,12 @@ module Gitlab
def update_webhooks(hooks, options)
hooks.each do |hook|
- sleep rate_limit_sleep_time if rate_limit_exceed?
client.edit_hook(repo, hook.id, hook.name, hook.config, options)
end
end
def restore_branches(branches)
branches.each do |name, sha|
- sleep rate_limit_sleep_time if rate_limit_exceed?
client.create_ref(repo, "refs/heads/#{name}", sha)
end
@@ -162,15 +117,12 @@ module Gitlab
def clean_up_restored_branches(branches)
branches.each do |name, _|
- sleep rate_limit_sleep_time if rate_limit_exceed?
client.delete_ref(repo, "heads/#{name}")
project.repository.rm_branch(project.creator, name)
end
end
def apply_labels(issuable)
- sleep rate_limit_sleep_time if rate_limit_exceed?
-
issue = client.issue(repo, issuable.iid)
if issue.labels.count > 0
@@ -183,12 +135,12 @@ module Gitlab
end
def import_comments(issuable)
- comments = paginate { client.issue_comments(repo, issuable.iid, per_page: 100) }
+ comments = client.issue_comments(repo, issuable.iid, per_page: 100)
create_comments(issuable, comments)
end
def import_comments_on_diff(merge_request)
- comments = paginate { client.pull_request_comments(repo, merge_request.iid, per_page: 100) }
+ comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100)
create_comments(merge_request, comments)
end