Backport JIRA service

CHANGELOG

@@ -19,6 +19,7 @@ v 8.3.0 (unreleased)
   - Fix 500 error when update group member permission
   - Trim leading and trailing whitespace of milestone and issueable titles (Jose Corcuera)
   - Recognize issue/MR/snippet/commit links as references
+  - Backport JIRA features from EE to CE
   - Add ignore whitespace change option to commit view
   - Fire update hook from GitLab
   - Style warning about mentioning many people in a comment

app/controllers/projects/services_controller.rb

 class Projects::ServicesController < Projects::ApplicationController
-  ALLOWED_PARAMS = [:title, :token, :type, :active, :api_key, :api_version, :subdomain,
+  ALLOWED_PARAMS = [:title, :token, :type, :active, :api_key, :api_url, :api_version, :subdomain,
                     :room, :recipients, :project_url, :webhook,
                     :user_key, :device, :priority, :sound, :bamboo_url, :username, :password,
                     :build_key, :server, :teamcity_url, :drone_url, :build_type,
@@ -10,7 +10,8 @@ class Projects::ServicesController < Projects::ApplicationController
                     :notify_only_broken_builds, :add_pusher,
                     :send_from_committer_email, :disable_diffs, :external_wiki_url,
                     :notify, :color,
-                    :server_host, :server_port, :default_irc_uri, :enable_ssl_verification]
+                    :server_host, :server_port, :default_irc_uri, :enable_ssl_verification,
+                    :jira_issue_transition_id]
   # Parameters to ignore if no value is specified
   FILTER_BLANK_PARAMS = [:password]

app/models/jira_issue.rb

+class JiraIssue < ExternalIssue
+end

app/models/merge_request.rb

@@ -335,7 +335,7 @@ class MergeRequest < ActiveRecord::Base
       issues = commits.flat_map { |c| c.closes_issues(current_user) }
       issues.push(*Gitlab::ClosingIssueExtractor.new(project, current_user).
                   closed_by_message(description))
-      issues.uniq
+      issues.uniq(&:id)
     else
       []
     end

app/models/project.rb

@@ -499,6 +499,10 @@ class Project < ActiveRecord::Base
     @ci_service ||= ci_services.find(&:activated?)
   end
+  def jira_tracker?
+    issues_tracker.to_param == 'jira'
+  end
+
   def avatar_type
     unless self.avatar.image?
       self.errors.add :avatar, 'only images allowed'
@@ -799,6 +803,10 @@ class Project < ActiveRecord::Base
     false
   end
+  def jira_tracker_active?
+    jira_tracker? && jira_service.active
+  end
+
   def ci_commit(sha)
     ci_commits.find_by(sha: sha)
   end

app/models/project_services/jira_service.rb

@@ -19,9 +19,24 @@
 #
 class JiraService < IssueTrackerService
+  include HTTParty
   include Gitlab::Application.routes.url_helpers
-  prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
+  DEFAULT_API_VERSION = 2
+
+  prop_accessor :username, :password, :api_url, :jira_issue_transition_id,
+                :title, :description, :project_url, :issues_url, :new_issue_url
+
+  before_validation :set_api_url, :set_jira_issue_transition_id
+
+  before_update :reset_password
+
+  def reset_password
+    # don't reset the password if a new one is provided
+    if api_url_changed? && !password_touched?
+      self.password = nil
+    end
+  end
   def help
     line1 = 'Setting `project_url`, `issues_url` and `new_issue_url` will '\
@@ -54,4 +69,228 @@ class JiraService < IssueTrackerService
   def to_param
     'jira'
   end
+
+  def fields
+    super.push(
+      { type: 'text', name: 'api_url', placeholder: 'https://jira.example.com/rest/api/2' },
+      { type: 'text', name: 'username', placeholder: '' },
+      { type: 'password', name: 'password', placeholder: '' },
+      { type: 'text', name: 'jira_issue_transition_id', placeholder: '2' }
+    )
+  end
+
+  def execute(push, issue = nil)
+    if issue.nil?
+      # No specific issue, that means
+      # we just want to test settings
+      test_settings
+    else
+      close_issue(push, issue)
+    end
+  end
+
+  def create_cross_reference_note(mentioned, noteable, author)
+    issue_name = mentioned.id
+    project = self.project
+    noteable_name = noteable.class.name.underscore.downcase
+    noteable_id = if noteable.is_a?(Commit)
+                    noteable.id
+                  else
+                    noteable.iid
+                  end
+
+    entity_url = build_entity_url(noteable_name.to_sym, noteable_id)
+
+    data = {
+      user: {
+        name: author.name,
+        url: resource_url(user_path(author)),
+      },
+      project: {
+        name: project.path_with_namespace,
+        url: resource_url(namespace_project_path(project.namespace, project))
+      },
+      entity: {
+        name: noteable_name.humanize.downcase,
+        url: entity_url
+      }
+    }
+
+    add_comment(data, issue_name)
+  end
+
+  def test_settings
+    result = JiraService.get(
+      jira_api_test_url,
+      headers: {
+        'Content-Type' => 'application/json',
+        'Authorization' => "Basic #{auth}"
+      }
+    )
+
+    case result.code
+    when 201, 200
+      Rails.logger.info("#{self.class.name} SUCCESS #{result.code}: Successfully connected to #{api_url}.")
+      true
+    else
+      Rails.logger.info("#{self.class.name} ERROR #{result.code}: #{result.parsed_response}")
+      false
+    end
+  rescue Errno::ECONNREFUSED => e
+    Rails.logger.info "#{self.class.name} ERROR: #{e.message}. API URL: #{api_url}."
+    false
+  end
+
+  private
+
+  def build_api_url_from_project_url
+    server = URI(project_url)
+    default_ports = [["http",80],["https",443]].include?([server.scheme,server.port])
+    server_url = "#{server.scheme}://#{server.host}"
+    server_url.concat(":#{server.port}") unless default_ports
+    "#{server_url}/rest/api/#{DEFAULT_API_VERSION}"
+  rescue
+    "" # looks like project URL was not valid
+  end
+
+  def set_api_url
+    self.api_url = build_api_url_from_project_url if self.api_url.blank?
+  end
+
+  def set_jira_issue_transition_id
+    self.jira_issue_transition_id ||= "2"
+  end
+
+  def close_issue(entity, issue)
+    commit_id = if entity.is_a?(Commit)
+                  entity.id
+                elsif entity.is_a?(MergeRequest)
+                  entity.last_commit.id
+                end
+    commit_url = build_entity_url(:commit, commit_id)
+
+    # Depending on the JIRA project's workflow, a comment during transition
+    # may or may not be allowed. Split the operation in to two calls so the
+    # comment always works.
+    transition_issue(issue)
+    add_issue_solved_comment(issue, commit_id, commit_url)
+  end
+
+  def transition_issue(issue)
+    message = {
+      transition: {
+        id: jira_issue_transition_id
+      }
+    }
+    send_message(close_issue_url(issue.iid), message.to_json)
+  end
+
+  def add_issue_solved_comment(issue, commit_id, commit_url)
+    comment = {
+      body: "Issue solved with [#{commit_id}|#{commit_url}]."
+    }
+
+    send_message(comment_url(issue.iid), comment.to_json)
+  end
+
+  def add_comment(data, issue_name)
+    url = comment_url(issue_name)
+    user_name = data[:user][:name]
+    user_url = data[:user][:url]
+    entity_name = data[:entity][:name]
+    entity_url = data[:entity][:url]
+    project_name = data[:project][:name]
+
+    message = {
+      body: "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]."
+    }
+
+    unless existing_comment?(issue_name, message[:body])
+      send_message(url, message.to_json)
+    end
+  end
+
+
+  def auth
+    require 'base64'
+    Base64.urlsafe_encode64("#{self.username}:#{self.password}")
+  end
+
+  def send_message(url, message)
+    result = JiraService.post(
+      url,
+      body: message,
+      headers: {
+        'Content-Type' => 'application/json',
+        'Authorization' => "Basic #{auth}"
+      }
+    )
+
+    message = case result.code
+              when 201, 200, 204
+                "#{self.class.name} SUCCESS #{result.code}: Successfully posted to #{url}."
+              when 401
+                "#{self.class.name} ERROR 401: Unauthorized. Check the #{self.username} credentials and JIRA access permissions and try again."
+              else
+                "#{self.class.name} ERROR #{result.code}: #{result.parsed_response}"
+              end
+
+    Rails.logger.info(message)
+    message
+  rescue URI::InvalidURIError, Errno::ECONNREFUSED => e
+    Rails.logger.info "#{self.class.name} ERROR: #{e.message}. Hostname: #{url}."
+  end
+
+  def existing_comment?(issue_name, new_comment)
+    result = JiraService.get(
+      comment_url(issue_name),
+      headers: {
+        'Content-Type' => 'application/json',
+        'Authorization' => "Basic #{auth}"
+      }
+    )
+
+    case result.code
+    when 201, 200
+      existing_comments = JSON.parse(result.body)['comments']
+
+      if existing_comments.present?
+        return existing_comments.map { |comment| comment['body'].include?(new_comment) }.any?
+      end
+    end
+
+    false
+  rescue JSON::ParserError
+    false
+  end
+
+  def resource_url(resource)
+    "#{Settings.gitlab['url'].chomp("/")}#{resource}"
+  end
+
+  def build_entity_url(entity_name, entity_id)
+    resource_url(
+      polymorphic_url(
+        [
+          self.project.namespace.becomes(Namespace),
+          self.project,
+          entity_name
+        ],
+        id: entity_id,
+        routing_type: :path
+      )
+    )
+  end
+
+  def close_issue_url(issue_name)
+    "#{self.api_url}/issue/#{issue_name}/transitions"
+  end
+
+  def comment_url(issue_name)
+    "#{self.api_url}/issue/#{issue_name}/comment"
+  end
+
+  def jira_api_test_url
+    "#{self.api_url}/myself"
+  end
 end

app/services/issues/close_service.rb

 module Issues
   class CloseService < Issues::BaseService
     def execute(issue, commit = nil)
+      if project.jira_tracker? && project.jira_service.active
+        project.jira_service.execute(commit, issue)
+        return issue
+      end
+
       if project.default_issues_tracker? && issue.close
         event_service.close_issue(issue, current_user)
         create_note(issue, commit)

app/services/system_note_service.rb

@@ -241,9 +241,14 @@ class SystemNoteService
       note_options.merge!(noteable: noteable)
     end
-    create_note(note_options)
+    if noteable.is_a?(ExternalIssue)
+      noteable.project.issues_tracker.create_cross_reference_note(noteable, mentioner, author)
+    else
+      create_note(note_options)
+    end
   end
+
   def self.cross_reference?(note_text)
     note_text.start_with?(cross_reference_note_prefix)
   end
@@ -259,7 +264,7 @@ class SystemNoteService
   #
   # Returns Boolean
   def self.cross_reference_disallowed?(noteable, mentioner)
-    return true if noteable.is_a?(ExternalIssue)
+    return true if noteable.is_a?(ExternalIssue) && !noteable.project.jira_tracker_active?
     return false unless mentioner.is_a?(MergeRequest)
     return false unless noteable.is_a?(Commit)

config/initializers/1_settings.rb

@@ -164,7 +164,7 @@ Settings.gitlab['signin_enabled'] ||= true if Settings.gitlab['signin_enabled'].
 Settings.gitlab['twitter_sharing_enabled'] ||= true if Settings.gitlab['twitter_sharing_enabled'].nil?
 Settings.gitlab['restricted_visibility_levels'] = Settings.send(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], [])
 Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username_changing_enabled'].nil?
-Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing)) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?))+)' if Settings.gitlab['issue_closing_pattern'].nil?
+Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing)) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z]*-\d*))+)' if Settings.gitlab['issue_closing_pattern'].nil?
 Settings.gitlab['default_projects_features'] ||= {}
 Settings.gitlab['webhook_timeout'] ||= 10
 Settings.gitlab['max_attachment_size'] ||= 10

db/migrate/20151012173029_set_jira_service_api_url.rb

+class SetJiraServiceApiUrl < ActiveRecord::Migration
+  # This migration can be performed online without errors, but some Jira API calls may be missed
+  # when doing so because api_url is not yet available.
+
+  def build_api_url_from_project_url(project_url, api_version)
+    # this is the exact logic previously used to build the Jira API URL from project_url
+    server = URI(project_url)
+    default_ports = [80, 443].include?(server.port)
+    server_url = "#{server.scheme}://#{server.host}"
+    server_url.concat(":#{server.port}") unless default_ports
+    "#{server_url}/rest/api/#{api_version}"
+  end
+
+  def get_api_version_from_api_url(api_url)
+    match = /\/rest\/api\/(?<api_version>\w+)$/.match(api_url)
+    match && match['api_version']
+  end
+
+  def change
+    reversible do |dir|
+      select_all("SELECT id, properties FROM services WHERE services.type IN ('JiraService')").each do |jira_service|
+        id = jira_service["id"]
+        properties = JSON.parse(jira_service["properties"])
+        properties_was = properties.clone
+
+        dir.up do
+          # remove api_version and set api_url
+          if properties['api_version'].present? && properties['project_url'].present?
+            begin
+              properties['api_url'] ||= build_api_url_from_project_url(properties['project_url'], properties['api_version'])
+            rescue
+              # looks like project_url was not a valid URL. Do nothing.
+            end
+          end
+          properties.delete('api_version') if properties.include?('api_version')
+        end
+
+        dir.down do
+          # remove api_url and set api_version (default to '2')
+          properties['api_version'] ||= get_api_version_from_api_url(properties['api_url']) || '2'
+          properties.delete('api_url') if properties.include?('api_url')
+        end
+
+        if properties != properties_was
+          execute("UPDATE services SET properties = '#{quote_string(properties.to_json)}' WHERE id = #{id}")
+        end
+      end
+    end
+  end
+end

doc/integration/README.md

@@ -4,6 +4,7 @@ GitLab integrates with multiple third-party services to allow external issue tra
 See the documentation below for details on how to configure these services.
+- [Jira](jira.md) Integrate with the JIRA issue tracker
 - [External issue tracker](external-issue-tracker.md) Redmine, JIRA, etc.
 - [LDAP](ldap.md) Set up sign in via LDAP
 - [OmniAuth](omniauth.md) Sign in via Twitter, GitHub, GitLab, and Google via OAuth.

doc/integration/jira.md

+# GitLab Jira integration
+
+GitLab can be configured to interact with Jira.
+Configuration happens via username and password.
+Connecting to a Jira server via CAS is not possible.
+
+Each project can be configured to connect to a different Jira instance, configuration is explained [here](#configuration).
+If you have one Jira instance you can pre-fill the settings page with a default template. To configure the template [see external issue tracker document](external-issue-tracker.md#service-template)).
+
+Once the project is connected to Jira, you can reference and close the issues in Jira directly from GitLab.
+
+
+## Table of Contents
+
+* [Referencing Jira Issues from GitLab](#referencing-jira-issues)
+* [Closing Jira Issues from GitLab](#closing-jira-issues)
+* [Configuration](#configuration)
+
+### Referencing Jira Issues
+
+When GitLab project has Jira issue tracker configured and enabled, mentioning Jira issue in GitLab will automatically add a comment in Jira issue with the link back to GitLab. This means that in comments in merge requests and commits referencing an issue, eg. `PROJECT-7`, will add a comment in Jira issue in the format:
+
+
+```
+ USER mentioned this issue in LINK_TO_THE_MENTION
+```
+
+* `USER` A user that mentioned the issue. This is the link to the user profile in GitLab.
+* `LINK_TO_THE_MENTION` Link to the origin of mention with a name of the entity where Jira issue was mentioned.
+Can be commit or merge request.
+
+
+![example of mentioning or closing the Jira issue](jira_issue_reference.png)
+
+
+### Closing Jira Issues
+
+Jira issues can be closed directly from GitLab by using trigger words, eg. `Resolves PROJECT-1`, `Closes PROJECT-1` or `Fixes PROJECT-1`, in commits and merge requests.
+When a commit which contains the trigger word in the commit message is pushed, GitLab will add a comment in the mentioned Jira issue.
+
+For example, for project named PROJECT in Jira, we implemented a new feature and created a merge request in GitLab.
+
+This feature was requested in Jira issue PROJECT-7. Merge request in GitLab contains the improvement and in merge request description we say that this merge request `Closes PROJECT-7` issue.
+
+Once this merge request is merged, Jira issue will be automatically closed with a link to the commit that resolved the issue.
+
+![A Git commit that causes the Jira issue to be closed](merge_request_close_jira.png)
+
+
+![The GitLab integration user leaves a comment on Jira](jira_service_close_issue.png)
+
+
+## Configuration
+
+### Configuring JIRA
+
+We need to create a user in JIRA which will have access to all projects that need to integrate with GitLab.
+Login to your JIRA instance as admin and under Administration go to User Management and create a new user.
+As an example, we'll create a user named `gitlab` and add it to `jira-developers` group.
+
+**It is important that the user `gitlab` has write-access to projects in JIRA**
+
+### Configuring GitLab
+
+### GitLab 7.8 EE and up with JIRA v6.x
+
+To enable JIRA integration in a project, navigate to the project Settings page and go to Services. Here you will find JIRA.
+
+Fill in the required details on the page:
+
+![Jira service page](jira_service_page.png)
+
+* `description` A name for the issue tracker (to differentiate between instances, for instance).
+* `project url` The URL to the JIRA project which is being linked to this GitLab project.
+* `issues url` The URL to the JIRA project issues overview for the project that is linked to this GitLab project.
+* `new issue url` This is the URL to create a new issue in JIRA for the project linked to this GitLab project.
+* `api url` The base URL of the JIRA API. It may be omitted, in which case GitLab will automatically use API version `2` based on the `project url`, i.e. `https://jira.example.com/rest/api/2`.
+* `username` The username of the user created in [configuring JIRA step](#configuring-jira).
+* `password` The password of the user created in [configuring JIRA step](#configuring-jira).
+* `Jira issue transition` This is the id of a transition that moves issues to a closed state. You can find this number under [JIRA workflow administration, see screenshot](jira_workflow_screenshot.png).  By default, this id is `2`. (In the example image, this is `2` as well)
+
+After saving the configuration, your GitLab project will be able to interact with the linked JIRA project.
+
+
+### GitLab 6.x-7.7 with JIRA v6.x
+
+**Note: GitLab 7.8 and up contain various integration improvements. We strongly recommend upgrading.**
+
+
+In `gitlab.yml` enable [JIRA issue tracker section by uncommenting the lines](https://gitlab.com/subscribers/gitlab-ee/blob/6-8-stable-ee/config/gitlab.yml.example#L111-115).
+This will make sure that all issues within GitLab are pointing to the JIRA issue tracker.
+
+We can also enable JIRA service that will allow us to interact with JIRA issues.
+
+For example, we can close issues in JIRA by a commit in GitLab.
+
+Go to project settings page and fill in the project name for the JIRA project:
+
+![Set the JIRA project name in GitLab to 'NEW'](jira_project_name.png)
+
+Next, go to the services page and find JIRA.
+
+![Jira services page](jira_service.png)
+
+1. Tick the active check box to enable the service.
+1. Supply the url to JIRA server, for example http://jira.sample
+1. Supply the username of a user we created under `Configuring JIRA` section, for example `gitlab`
+1. Supply the password of the user
+1. Optional: supply the JIRA api version, default is version
+1. Optional: supply the JIRA issue transition ID (issue transition to closed). This is dependant on JIRA settings, default is 2
+1. Save
+
+Now we should be able to interact with JIRA issues.

features/project/service.feature

@@ -55,6 +55,12 @@ Feature: Project Services
     And I fill email on push settings
     Then I should see email on push service settings saved
+  Scenario: Activate JIRA service
+    When I visit project "Shop" services page
+    And I click jira service link
+    And I fill jira settings
+    Then I should see jira service settings saved
+
   Scenario: Activate Irker (IRC Gateway) service
     When I visit project "Shop" services page
     And I click Irker service link

features/steps/project/services.rb

@@ -173,6 +173,24 @@ class Spinach::Features::ProjectServices < Spinach::FeatureSteps
     expect(find_field('Sound').find('option[selected]').value).to eq 'bike'
   end
+  step 'I click jira service link' do
+    click_link 'JIRA'
+  end
+
+  step 'I fill jira settings' do
+    fill_in 'Project url', with: 'http://jira.example'
+    fill_in 'Username', with: 'gitlab'
+    fill_in 'Password', with: 'gitlab'
+    fill_in 'Api url', with: 'http://jira.example/rest/api/2'
+    click_button 'Save'
+  end
+
+  step 'I should see jira service settings saved' do
+    expect(find_field('Project url').value).to eq 'http://jira.example'
+    expect(find_field('Username').value).to eq 'gitlab'
+    expect(find_field('Api url').value).to eq 'http://jira.example/rest/api/2'
+  end
+
   step 'I click Atlassian Bamboo CI service link' do
     click_link 'Atlassian Bamboo CI'
   end