Skip to content
Snippets Groups Projects
Commit dbcbbf26 authored by Ahmad Sherif's avatar Ahmad Sherif
Browse files

Speed up label-applying process for GitHub importing

* No need to re-fetch issues from GH to read their labels, the labels
  are already there from the index request.
* No need to look up labels on the database for every application, so we
  cache them.
parent 395a9301
No related branches found
No related tags found
No related merge requests found
Loading
@@ -10,6 +10,7 @@ module Gitlab
Loading
@@ -10,6 +10,7 @@ module Gitlab
@repo = project.import_source @repo = project.import_source
@repo_url = project.import_url @repo_url = project.import_url
@errors = [] @errors = []
@labels = {}
   
if credentials if credentials
@client = Client.new(credentials[:user]) @client = Client.new(credentials[:user])
Loading
@@ -49,7 +50,8 @@ module Gitlab
Loading
@@ -49,7 +50,8 @@ module Gitlab
client.labels(repo, per_page: 100) do |labels| client.labels(repo, per_page: 100) do |labels|
labels.each do |raw| labels.each do |raw|
begin begin
LabelFormatter.new(project, raw).create! label = LabelFormatter.new(project, raw).create!
@labels[label.title] = label.id
rescue => e rescue => e
errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
end end
Loading
@@ -77,7 +79,7 @@ module Gitlab
Loading
@@ -77,7 +79,7 @@ module Gitlab
if gh_issue.valid? if gh_issue.valid?
begin begin
issue = gh_issue.create! issue = gh_issue.create!
apply_labels(issue) apply_labels(issue, raw)
import_comments(issue) if gh_issue.has_comments? import_comments(issue) if gh_issue.has_comments?
rescue => e rescue => e
errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
Loading
@@ -98,7 +100,7 @@ module Gitlab
Loading
@@ -98,7 +100,7 @@ module Gitlab
restore_target_branch(pull_request) unless pull_request.target_branch_exists? restore_target_branch(pull_request) unless pull_request.target_branch_exists?
   
merge_request = pull_request.create! merge_request = pull_request.create!
apply_labels(merge_request) apply_labels(merge_request, raw)
import_comments(merge_request) import_comments(merge_request)
import_comments_on_diff(merge_request) import_comments_on_diff(merge_request)
rescue => e rescue => e
Loading
@@ -131,12 +133,10 @@ module Gitlab
Loading
@@ -131,12 +133,10 @@ module Gitlab
project.repository.after_remove_branch project.repository.after_remove_branch
end end
   
def apply_labels(issuable) def apply_labels(issuable, raw_issuable)
issue = client.issue(repo, issuable.iid) if raw_issuable.labels.count > 0
label_ids = raw_issuable.labels
if issue.labels.count > 0 .map { |attrs| @labels[attrs.name] }
label_ids = issue.labels
.map { |attrs| project.labels.find_by(title: attrs.name).try(:id) }
.compact .compact
   
issuable.update_attribute(:label_ids, label_ids) issuable.update_attribute(:label_ids, label_ids)
Loading
Loading
Loading
@@ -57,7 +57,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
Loading
@@ -57,7 +57,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
created_at: created_at, created_at: created_at,
updated_at: updated_at, updated_at: updated_at,
closed_at: nil, closed_at: nil,
url: 'https://api.github.com/repos/octocat/Hello-World/issues/1347' url: 'https://api.github.com/repos/octocat/Hello-World/issues/1347',
labels: [double(name: 'Label #1')],
) )
end end
   
Loading
@@ -75,7 +76,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
Loading
@@ -75,7 +76,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
created_at: created_at, created_at: created_at,
updated_at: updated_at, updated_at: updated_at,
closed_at: nil, closed_at: nil,
url: 'https://api.github.com/repos/octocat/Hello-World/issues/1348' url: 'https://api.github.com/repos/octocat/Hello-World/issues/1348',
labels: [double(name: 'Label #2')],
) )
end end
   
Loading
@@ -94,7 +96,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
Loading
@@ -94,7 +96,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
updated_at: updated_at, updated_at: updated_at,
closed_at: nil, closed_at: nil,
merged_at: nil, merged_at: nil,
url: 'https://api.github.com/repos/octocat/Hello-World/pulls/1347' url: 'https://api.github.com/repos/octocat/Hello-World/pulls/1347',
labels: [double(name: 'Label #3')],
) )
end end
   
Loading
@@ -148,9 +151,7 @@ describe Gitlab::GithubImport::Importer, lib: true do
Loading
@@ -148,9 +151,7 @@ describe Gitlab::GithubImport::Importer, lib: true do
errors: [ errors: [
{ type: :label, url: "https://api.github.com/repos/octocat/Hello-World/labels/bug", errors: "Validation failed: Title can't be blank, Title is invalid" }, { type: :label, url: "https://api.github.com/repos/octocat/Hello-World/labels/bug", errors: "Validation failed: Title can't be blank, Title is invalid" },
{ type: :milestone, url: "https://api.github.com/repos/octocat/Hello-World/milestones/1", errors: "Validation failed: Title has already been taken" }, { type: :milestone, url: "https://api.github.com/repos/octocat/Hello-World/milestones/1", errors: "Validation failed: Title has already been taken" },
{ type: :issue, url: "https://api.github.com/repos/octocat/Hello-World/issues/1347", errors: "Invalid Repository. Use user/repo format." },
{ type: :issue, url: "https://api.github.com/repos/octocat/Hello-World/issues/1348", errors: "Validation failed: Title can't be blank, Title is too short (minimum is 0 characters)" }, { type: :issue, url: "https://api.github.com/repos/octocat/Hello-World/issues/1348", errors: "Validation failed: Title can't be blank, Title is too short (minimum is 0 characters)" },
{ type: :pull_request, url: "https://api.github.com/repos/octocat/Hello-World/pulls/1347", errors: "Invalid Repository. Use user/repo format." },
{ type: :pull_request, url: "https://api.github.com/repos/octocat/Hello-World/pulls/1347", errors: "Validation failed: Validate branches Cannot Create: This merge request already exists: [\"New feature\"]" }, { type: :pull_request, url: "https://api.github.com/repos/octocat/Hello-World/pulls/1347", errors: "Validation failed: Validate branches Cannot Create: This merge request already exists: [\"New feature\"]" },
{ type: :wiki, errors: "Gitlab::Shell::Error" }, { type: :wiki, errors: "Gitlab::Shell::Error" },
{ type: :release, url: 'https://api.github.com/repos/octocat/Hello-World/releases/2', errors: "Validation failed: Description can't be blank" } { type: :release, url: 'https://api.github.com/repos/octocat/Hello-World/releases/2', errors: "Validation failed: Description can't be blank" }
Loading
Loading
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment