Merge branch '49231-import-issues-csv' into 'master'

Import issues from CSV

Closes #49231

See merge request gitlab-org/gitlab-ce!23532

app/assets/stylesheets/framework/modal.scss

@@ -101,3 +101,41 @@ body.modal-open {
     margin: 0;
   }
 }
+
+.issues-import-modal,
+.issues-export-modal {
+  .modal-header {
+    justify-content: flex-start;
+
+    .import-export-svg-container {
+      flex-grow: 1;
+      height: 56px;
+      padding: $gl-btn-padding $gl-btn-padding 0;
+
+      > svg {
+        float: right;
+        height: 100%;
+      }
+    }
+  }
+
+  .modal-body {
+    padding: 0;
+
+    .modal-subheader {
+      justify-content: flex-start;
+      align-items: center;
+      border-bottom: 1px solid $modal-border-color;
+      padding: 14px;
+    }
+
+    .modal-text {
+      padding: $gl-padding-24 $gl-padding;
+      min-height: $modal-body-height;
+    }
+  }
+
+  .checkmark {
+    color: $green-400;
+  }
+}

app/assets/stylesheets/framework/variables.scss

@@ -656,6 +656,7 @@ $border-color-settings: #e1e1e1;
 Modals
 */
 $modal-body-height: 134px;
+$modal-border-color: #e9ecef;
 $priority-label-empty-state-width: 114px;

app/assets/stylesheets/pages/issues.scss

@@ -155,6 +155,14 @@ ul.related-merge-requests > li {
   }
 }
+.issues-nav-controls {
+  font-size: 0;
+
+  .btn-group:empty {
+    display: none;
+  }
+}
+
 .issuable-email-modal-btn {
   padding: 0;
   color: $blue-600;

app/controllers/concerns/uploads_actions.rb

@@ -7,12 +7,12 @@ module UploadsActions
   UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo favicon).freeze
   def create
-    link_to_file = UploadService.new(model, params[:file], uploader_class).execute
+    uploader = UploadService.new(model, params[:file], uploader_class).execute
     respond_to do |format|
-      if link_to_file
+      if uploader
         format.json do
-          render json: { link: link_to_file }
+          render json: { link: uploader.to_h }
         end
       else
         format.json do

app/controllers/projects/issues_controller.rb

@@ -10,7 +10,7 @@ class Projects::IssuesController < Projects::ApplicationController
   include SpammableActions
   def self.issue_except_actions
-    %i[index calendar new create bulk_update]
+    %i[index calendar new create bulk_update import_csv]
   end
   def self.set_issuables_index_only_actions
@@ -37,6 +37,8 @@ class Projects::IssuesController < Projects::ApplicationController
   # Allow create a new branch and empty WIP merge request from current issue
   before_action :authorize_create_merge_request_from!, only: [:create_merge_request]
+  before_action :authorize_import_issues!, only: [:import_csv]
+
   before_action :set_suggested_issues_feature_flags, only: [:new]
   respond_to :html
@@ -175,6 +177,20 @@ class Projects::IssuesController < Projects::ApplicationController
     end
   end
+  def import_csv
+    return render_404 unless Feature.enabled?(:issues_import_csv)
+
+    if uploader = UploadService.new(project, params[:file]).execute
+      ImportIssuesCsvWorker.perform_async(current_user.id, project.id, uploader.upload.id)
+
+      flash[:notice] = _("Your issues are being imported. Once finished, you'll get a confirmation email.")
+    else
+      flash[:alert] = _("File upload error.")
+    end
+
+    redirect_to project_issues_path(project)
+  end
+
   protected
   # rubocop: disable CodeReuse/ActiveRecord

app/mailers/emails/issues.rb

@@ -77,6 +77,17 @@ module Emails
       mail_answer_thread(issue, issue_thread_options(updated_by_user.id, recipient.id, reason))
     end
+    def import_issues_csv_email(user_id, project_id, results)
+      @user = User.find(user_id)
+      @project = Project.find(project_id)
+      @results = results
+
+      mail(to: @user.notification_email, subject: subject('Imported issues')) do |format|
+        format.html { render layout: 'mailer' }
+        format.text { render layout: 'mailer' }
+      end
+    end
+
     private
     def setup_issue_mail(issue_id, recipient_id)

app/mailers/previews/notify_preview.rb

@@ -76,6 +76,10 @@ class NotifyPreview < ActionMailer::Preview
     Notify.changed_milestone_issue_email(user.id, issue.id, milestone, user.id)
   end
+  def import_issues_csv_email
+    Notify.import_issues_csv_email(user, project, { success: 3, errors: [5, 6, 7], valid_file: true })
+  end
+
   def closed_merge_request_email
     Notify.closed_merge_request_email(user.id, issue.id, user.id).message
   end

app/policies/project_policy.rb

@@ -222,6 +222,8 @@ class ProjectPolicy < BasePolicy
   rule { owner | admin | guest | group_member }.prevent :request_access
   rule { ~request_access_enabled }.prevent :request_access
+  rule { can?(:developer_access) & can?(:create_issue) }.enable :import_issues
+
   rule { can?(:developer_access) }.policy do
     enable :admin_merge_request
     enable :admin_milestone

app/services/issues/import_csv_service.rb

+# frozen_string_literal: true
+
+module Issues
+  class ImportCsvService
+    def initialize(user, project, csv_io)
+      @user = user
+      @project = project
+      @csv_io = csv_io
+      @results = { success: 0, error_lines: [], parse_error: false }
+    end
+
+    def execute
+      process_csv
+      email_results_to_user
+
+      @results
+    end
+
+    private
+
+    def process_csv
+      csv_data = @csv_io.open(&:read).force_encoding(Encoding::UTF_8)
+
+      CSV.new(csv_data, col_sep: detect_col_sep(csv_data.lines.first), headers: true).each.with_index(2) do |row, line_no|
+        issue = Issues::CreateService.new(@project, @user, title: row[0], description: row[1]).execute
+
+        if issue.persisted?
+          @results[:success] += 1
+        else
+          @results[:error_lines].push(line_no)
+        end
+      end
+    rescue ArgumentError, CSV::MalformedCSVError
+      @results[:parse_error] = true
+    end
+
+    def email_results_to_user
+      Notify.import_issues_csv_email(@user.id, @project.id, @results).deliver_now
+    end
+
+    def detect_col_sep(header)
+      if header.include?(",")
+        ","
+      elsif header.include?(";")
+        ";"
+      elsif header.include?("\t")
+        "\t"
+      else
+        raise CSV::MalformedCSVError
+      end
+    end
+  end
+end

app/services/upload_service.rb

@@ -11,7 +11,7 @@ class UploadService
     uploader = @uploader_class.new(@model, nil, @uploader_context)
     uploader.store!(@file)
-    uploader.to_h
+    uploader
   end
   private

app/views/notify/import_issues_csv_email.html.haml

+- text_style = 'font-size:16px; text-align:center;  line-height:30px;'
+
+%p{ style: text_style }
+  Your CSV import for project
+  %a{ href: project_url(@project), style: "color:#3777b0; text-decoration:none;" }
+    = @project.full_name
+  has been completed.
+
+%p{ style: text_style }
+  #{pluralize(@results[:success], 'issue')} imported.
+
+- if @results[:error_lines].present?
+  %p{ style: text_style }
+    Errors found on line #{'number'.pluralize(@results[:error_lines].size)}: #{@results[:error_lines].join(', ')}. Please check if these lines have an issue title.
+
+- if @results[:parse_error]
+  %p{ style: text_style }
+    Error parsing CSV file. Please make sure it has the correct format: a delimited text file that uses a comma to separate values.

app/views/notify/import_issues_csv_email.text.erb

+Your CSV import for project <%= @project.full_name %> (<%= project_url(@project) %>) has been completed.
+
+<%= pluralize(@results[:success], 'issue') %> imported.
+
+<% if @results[:error_lines].present? %>
+Errors found on line <%= 'number'.pluralize(@results[:error_lines].size) %>: <%= @results[:error_lines].join(', ') %>. Please check if these lines have an issue title.
+<% end %>
+
+<% if @results[:parse_error] %>
+Error parsing CSV file. Please make sure it has the correct format: a delimited text file that uses a comma to separate values.
+<% end %>

app/views/projects/cleanup/_show.html.haml

@@ -24,6 +24,6 @@
             = _("No file selected")
           = f.file_field :bfg_object_map, accept: 'text/plain', class: "hidden js-object-map-input", required: true
         .form-text.text-muted
-          = _("The maximum file size allowed is %{max_attachment_size}mb") % { max_attachment_size: Gitlab::CurrentSettings.max_attachment_size }
+          = _("The maximum file size allowed is %{size}.") % { size: number_to_human_size(Gitlab::CurrentSettings.max_attachment_size.megabytes) }
       = f.submit _('Start cleanup'), class: 'btn btn-success'

app/views/projects/issues/_import_export.svg

+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 238 111" xmlns:xlink="http://www.w3.org/1999/xlink"><defs><rect id="4" width="82" rx="3" height="28" fill="#fff"/><path id="5" d="m68.926 12.09v-2.41c0-.665-.437-.888-.975-.507l-6.552 4.631c-.542.383-.539.998 0 1.379l6.552 4.631c.542.383.975.154.975-.507v-2.41h4.874c.668 0 1.2-.538 1.2-1.201v-2.406c0-.668-.537-1.201-1.2-1.201h-4.874" fill="#fc8a51"/><path id="6" d="m4 24h74v-20h-74v20m-4-21c0-1.655 1.338-2.996 2.991-2.996h76.02c1.652 0 2.991 1.35 2.991 2.996v22.01c0 1.655-1.338 2.996-2.991 2.996h-76.02c-1.652 0-2.991-1.35-2.991-2.996v-22.01"/><circle id="2" cx="16" cy="14" r="7"/><circle id="0" cx="16" cy="14" r="7"/><mask id="3" width="14" height="14" x="0" y="0" fill="#fff"><use xlink:href="#2"/></mask><mask id="1" width="14" height="14" x="0" y="0" fill="#fff"><use xlink:href="#0"/></mask></defs><g fill="none" fill-rule="evenodd"><rect width="98" height="111" fill="#fff" rx="6"/><path fill="#e5e5e5" fill-rule="nonzero" d="m4 6.01v98.99c0 1.11.897 2.01 2 2.01h85.998c1.105 0 2-.897 2-2.01v-98.99c0-1.11-.897-2.01-2-2.01h-85.998c-1.105 0-2 .897-2 2.01m-4 0c0-3.318 2.685-6.01 6-6.01h85.998c3.314 0 6 2.689 6 6.01v98.99c0 3.318-2.685 6.01-6 6.01h-85.998c-3.314 0-6-2.689-6-6.01v-98.99"/><rect width="76" height="85" x="11" y="12" fill="#f9f9f9" rx="3"/><g transform="translate(37 59)"><use xlink:href="#4"/><path fill="#e5e5e5" fill-rule="nonzero" d="m4 24h74v-20h-74v20m-4-21c0-1.655 1.338-2.996 2.991-2.996h76.02c1.652 0 2.991 1.35 2.991 2.996v22.01c0 1.655-1.338 2.996-2.991 2.996h-76.02c-1.652 0-2.991-1.35-2.991-2.996v-22.01"/><use fill="#fff" stroke="#6b4fbb" stroke-width="8" mask="url(#1)" xlink:href="#0"/><use xlink:href="#5"/></g><g transform="translate(140)"><path fill="#fff" d="m0 4h94v103h-94z"/><path fill="#e5e5e5" fill-rule="nonzero" d="m0 74v30.993c0 3.318 2.687 6.01 6 6.01h85.998c3.316 0 6-2.69 6-6.01v-98.99c0-3.318-2.687-6.01-6-6.01h-85.998c-3.316 0-6 2.69-6 6.01v.993h4v-.993c0-1.11.896-2.01 2-2.01h85.998c1.105 0 2 .897 2 2.01v98.99c0 1.11-.896 2.01-2 2.01h-85.998c-1.105 0-2-.897-2-2.01v-30.993h-4"/><g fill="#f9f9f9"><rect width="82" height="28" x="8" y="12" rx="3"/><rect width="82" height="28" x="8" y="43" rx="3"/></g></g><g fill-rule="nonzero" transform="translate(148 73)"><use fill="#e5e5e5" xlink:href="#6"/><path fill="#6b4fbb" d="m17 17c1.657 0 3-1.343 3-3 0-1.657-1.343-3-3-3-1.657 0-3 1.343-3 3 0 1.657 1.343 3 3 3m0 4c-3.866 0-7-3.134-7-7 0-3.866 3.134-7 7-7 3.866 0 7 3.134 7 7 0 3.866-3.134 7-7 7"/></g><g transform="translate(25 24)"><use xlink:href="#4"/><use fill="#e5e5e5" fill-rule="nonzero" xlink:href="#6"/><use fill="#fff" stroke="#6b4fbb" stroke-width="8" mask="url(#3)" xlink:href="#2"/><use xlink:href="#5"/></g><g transform="translate(107 10)"><use xlink:href="#4"/><use fill="#fc8a51" fill-opacity=".3" fill-rule="nonzero" xlink:href="#6"/><path fill="#6b4fbb" fill-rule="nonzero" d="m16 17c1.657 0 3-1.343 3-3 0-1.657-1.343-3-3-3-1.657 0-3 1.343-3 3 0 1.657 1.343 3 3 3m0 4c-3.866 0-7-3.134-7-7 0-3.866 3.134-7 7-7 3.866 0 7 3.134 7 7 0 3.866-3.134 7-7 7" id="7"/><use xlink:href="#5"/></g><g transform="translate(128 41)"><use xlink:href="#4"/><use fill="#fc8a51" fill-opacity=".3" fill-rule="nonzero" xlink:href="#6"/><use xlink:href="#7"/><path fill="#fc8a51" d="m66.926 12.09v-2.41c0-.665-.437-.888-.975-.507l-6.552 4.631c-.542.383-.539.998 0 1.379l6.552 4.631c.542.383.975.154.975-.507v-2.41h4.874c.668 0 1.2-.538 1.2-1.201v-2.406c0-.668-.537-1.201-1.2-1.201h-4.874"/></g></g></svg>
\ No newline at end of file

app/views/projects/issues/_nav_btns.html.haml

-= render 'shared/issuable/feed_buttons'
-
-- if @can_bulk_update
-  = button_tag "Edit issues", class: "btn btn-default append-right-10 js-bulk-update-toggle"
-- if show_new_issue_link?(@project)
-  = link_to "New issue", new_project_issue_path(@project,
-                                                issue: { assignee_id: finder.assignee.try(:id),
-                                                         milestone_id: finder.milestones.first.try(:id) }),
-                                                class: "btn btn-success",
-                                                title: "New issue",
-                                                id: "new_issue_link"
+- show_feed_buttons = local_assigns.fetch(:show_feed_buttons, true)
+- show_import_button = local_assigns.fetch(:show_import_button, true) && Feature.enabled?(:issues_import_csv) && can?(current_user, :import_issues, @project)
+- show_export_button = local_assigns.fetch(:show_export_button, true)
+
+.nav-controls.issues-nav-controls
+  - if show_feed_buttons
+    = render 'shared/issuable/feed_buttons'
+
+  .btn-group.append-right-10<
+    - if show_export_button
+      = render_if_exists 'projects/issues/export_csv/button'
+
+    - if show_import_button
+      = render 'projects/issues/import_csv/button'
+
+  - if @can_bulk_update
+    = button_tag _("Edit issues"), class: "btn btn-default append-right-10 js-bulk-update-toggle"
+  - if show_new_issue_link?(@project)
+    = link_to _("New issue"), new_project_issue_path(@project,
+                                                  issue: { assignee_id: finder.assignee.try(:id),
+                                                           milestone_id: finder.milestones.first.try(:id) }),
+                                                  class: "btn btn-success",
+                                                  title: _("New issue"),
+                                                  id: "new_issue_link"
+
+- if show_export_button
+  = render_if_exists 'projects/issues/export_csv/modal'
+
+- if show_import_button
+  = render 'projects/issues/import_csv/modal'

app/views/projects/issues/import_csv/_button.html.haml

+- type = local_assigns.fetch(:type, :icon)
+
+%button.csv-import-button.btn{ title: _('Import CSV'), class: ('has-tooltip' if type == :icon),
+  data: { toggle: 'modal', target: '.issues-import-modal' } }
+  - if type == :icon
+    = sprite_icon('upload')
+  - else
+    = _('Import CSV')
+

app/views/projects/issues/import_csv/_modal.html.haml

+.issues-import-modal.modal
+  .modal-dialog
+    .modal-content
+      = form_tag import_csv_namespace_project_issues_path, multipart: true do
+        .modal-header
+          %h3
+            = _('Import issues')
+          .import-export-svg-container
+            = render 'projects/issues/import_export.svg'
+          %a.close{ href: '#', 'data-dismiss' => 'modal' } ×
+        .modal-body
+          .modal-text
+            %p
+              = _("Your issues will be imported in the background. Once finished, you'll get a confirmation email.")
+            .form-group
+              = label_tag :file, _('Upload CSV file'), class: 'label-bold'
+              %div
+                = file_field_tag :file, accept: '.csv,text/csv', required: true
+            %p.text-secondary
+              = _('It must have a header row and at least two columns: the first column is the issue title and the second column is the issue description. The separator is automatically detected.')
+              = _('The maximum file size allowed is %{size}.') % { size: number_to_human_size(Gitlab::CurrentSettings.max_attachment_size.megabytes) }
+        .modal-footer
+          %button{ type: 'submit', class: 'btn btn-success', title: _('Import issues') }
+            = _('Import issues')

app/views/projects/issues/index.html.haml

@@ -11,8 +11,7 @@
   %div{ class: (container_class) }
     .top-area
       = render 'shared/issuable/nav', type: :issues
-      .nav-controls
-        = render "projects/issues/nav_btns"
+      = render "projects/issues/nav_btns"
     = render 'shared/issuable/search_bar', type: :issues
     - if @can_bulk_update
@@ -23,4 +22,4 @@
       - if new_issue_email
         = render 'projects/issuable_by_email', email: new_issue_email, issuable_type: 'issue'
 - else
-  = render 'shared/empty_states/issues', button_path: new_project_issue_path(@project)
+  = render 'shared/empty_states/issues', button_path: new_project_issue_path(@project), show_import_button: true

app/views/shared/empty_states/_issues.html.haml

 - button_path = local_assigns.fetch(:button_path, false)
 - project_select_button = local_assigns.fetch(:project_select_button, false)
+- show_import_button = local_assigns.fetch(:show_import_button, false) && Feature.enabled?(:issues_import_csv) && can?(current_user, :import_issues, @project)
 - has_button = button_path || project_select_button
 .row.empty-state
@@ -21,12 +22,20 @@
         - if has_button
           .text-center
             - if project_select_button
-              = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue', type: :issues, with_feature_enabled: 'issues'
+              = render 'shared/new_project_item_select', path: 'issues/new', label: _('New issue'), type: :issues, with_feature_enabled: 'issues'
             - else
-              = link_to 'New issue', button_path, class: 'btn btn-success', title: 'New issue', id: 'new_issue_link'
+              = link_to _('New issue'), button_path, class: 'btn btn-success', title: _('New issue'), id: 'new_issue_link'
+
+            - if show_import_button
+              = render 'projects/issues/import_csv/button', type: :text
+
       - else
         %h4.text-center= _("There are no issues to show")
         %p
           = _("The Issue Tracker is the place to add things that need to be improved or solved in a project. You can register or sign in to create issues for this project.")
         .text-center
           = link_to _('Register / Sign In'), new_user_session_path, class: 'btn btn-success'
+
+- if show_import_button
+  = render 'projects/issues/import_csv/modal'
+

app/views/shared/issuable/_feed_buttons.html.haml

-= link_to safe_params.merge(rss_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: 'Subscribe to RSS feed' do
+= link_to safe_params.merge(rss_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: _('Subscribe to RSS feed') do
   = icon('rss')
-= link_to safe_params.merge(calendar_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: 'Subscribe to calendar' do
+= link_to safe_params.merge(calendar_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: _('Subscribe to calendar') do
   = custom_icon('icon_calendar')