Finish updates to use JIRA gem

Code improvements, bug fixes, finish documentation and specs

CHANGELOG.md

@@ -161,6 +161,7 @@ Please view this file on the master branch, on stable branches it's out of date.
   - Make searching for commits case insensitive
   - Fix resolved discussion display in side-by-side diff view !6575
   - Optimize GitHub importing for speed and memory
+  - Refactor Jira service to use jira-ruby gem
   - API: expose pipeline data in builds API (!6502, Guilherme Salazar)
   - Notify the Merger about merge after successful build (Dimitris Karakasilis)
   - Reduce queries needed to find users using their SSH keys when pushing commits

Gemfile

@@ -162,7 +162,7 @@ gem 'connection_pool', '~> 2.0'
 gem 'hipchat', '~> 1.5.0'
 # JIRA integration
-gem 'jira-ruby', '~> 0.1.17'
+gem 'jira-ruby', '~> 1.1.0'
 # Flowdock integration
 gem 'gitlab-flowdock-git-hook', '~> 1.0.1'

Gemfile.lock

@@ -356,6 +356,9 @@ GEM
       cause
       json
     ipaddress (0.8.3)
+    jira-ruby (1.1.0)
+      activesupport
+      oauth (~> 0.5, >= 0.5.0)
     jquery-atwho-rails (1.3.2)
     jquery-rails (4.1.1)
       rails-dom-testing (>= 1, < 3)
@@ -421,7 +424,7 @@ GEM
       mini_portile2 (~> 2.1.0)
       pkg-config (~> 1.1.7)
     numerizer (0.1.1)
-    oauth (0.4.7)
+    oauth (0.5.1)
     oauth2 (1.2.0)
       faraday (>= 0.8, < 0.10)
       jwt (~> 1.0)
@@ -881,6 +884,7 @@ DEPENDENCIES
   html-pipeline (~> 1.11.0)
   httparty (~> 0.13.3)
   influxdb (~> 0.2)
+  jira-ruby (~> 1.1.0)
   jquery-atwho-rails (~> 1.3.2)
   jquery-rails (~> 4.1.0)
   jquery-turbolinks (~> 2.1.0)

app/models/project_services/bugzilla_service.rb

 class BugzillaService < IssueTrackerService
+  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
+
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
   def title

app/models/project_services/custom_issue_tracker_service.rb

 class CustomIssueTrackerService < IssueTrackerService
+  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
+
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
   def title

app/models/project_services/gitlab_issue_tracker_service.rb

 class GitlabIssueTrackerService < IssueTrackerService
   include Gitlab::Routing.url_helpers
+  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
+
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
   default_value_for :default, true

app/models/project_services/issue_tracker_service.rb

 class IssueTrackerService < Service
-  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
-
   default_value_for :category, 'issue_tracker'
   # Pattern used to extract links from comments
@@ -38,18 +36,24 @@ class IssueTrackerService < Service
     ]
   end
-  def initialize_properties
-    if properties.nil?
-      if enabled_in_gitlab_config
+  # Initialize with default properties values
+  # or receive a block with custom properties
+  def initialize_properties(&block)
+    return unless properties.nil?
+
+    if enabled_in_gitlab_config
+      if block_given?
+        yield
+      else
         self.properties = {
           title: issues_tracker['title'],
           project_url: issues_tracker['project_url'],
           issues_url: issues_tracker['issues_url'],
           new_issue_url: issues_tracker['new_issue_url']
         }
-      else
-        self.properties = {}
       end
+    else
+      self.properties = {}
     end
   end

app/models/project_services/jira_service.rb

@@ -18,19 +18,16 @@
 #  note_events           :boolean          default(TRUE), not null
 #  build_events          :boolean          default(FALSE), not null
 #
-require 'jira'
 class JiraService < IssueTrackerService
-  include HTTParty
-  include Gitlab::Application.routes.url_helpers
-  DEFAULT_API_VERSION = 2
+  include Gitlab::Routing.url_helpers
+  validates :url, url: true, presence: true, if: :activated?
+  validates :project_key, presence: true, if: :activated?
   prop_accessor :username, :password, :url, :project_key,
                 :jira_issue_transition_id, :title, :description
-  before_validation :set_jira_issue_transition_id
-
   before_update :reset_password
   # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1
@@ -38,6 +35,15 @@ class JiraService < IssueTrackerService
     @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)}
   end
+  def initialize_properties
+    super do
+      self.properties = {
+        title: issues_tracker['title'],
+        url: issues_tracker['url']
+      }
+    end
+  end
+
   def reset_password
     # don't reset the password if a new one is provided
     if url_changed? && !password_touched?
@@ -47,19 +53,20 @@ class JiraService < IssueTrackerService
   def options
     url = URI.parse(self.url)
+
     {
-      :username         => self.username,
-      :password         => self.password,
-      :site             => URI.join(url, '/').to_s,
-      :context_path     => url.path,
-      :auth_type        => :basic,
-      :read_timeout     => 120,
-      :use_ssl          => url.scheme == 'https'
+      username: self.username,
+      password: self.password,
+      site: URI.join(url, '/').to_s,
+      context_path: url.path,
+      auth_type: :basic,
+      read_timeout: 120,
+      use_ssl: url.scheme == 'https'
     }
   end
   def client
-    @client ||= ::JIRA::Client.new(options)
+    @client ||= JIRA::Client.new(options)
   end
   def jira_project
@@ -95,13 +102,14 @@ class JiraService < IssueTrackerService
   def fields
     [
       { type: 'text', name: 'url', title: 'URL', placeholder: 'https://jira.example.com' },
-      { type: 'text', name: 'project_key', placeholder: 'PROJ' },
+      { type: 'text', name: 'project_key', placeholder: 'Project Key' },
       { type: 'text', name: 'username', placeholder: '' },
       { type: 'password', name: 'password', placeholder: '' },
       { type: 'text', name: 'jira_issue_transition_id', placeholder: '2' }
     ]
   end
+  # URLs to redirect from Gitlab issues pages to jira issue tracker
   def project_url
     "#{url}/issues/?jql=project=#{project_key}"
   end
@@ -147,7 +155,8 @@ class JiraService < IssueTrackerService
       },
       entity: {
         name: noteable_name.humanize.downcase,
-        url: entity_url
+        url: entity_url,
+        title: noteable.title
       }
     }
@@ -155,27 +164,24 @@ class JiraService < IssueTrackerService
   end
   def test_settings
-    return unless api_utrl.present?
+    return unless url.present?
     # Test settings by getting the project
     jira_project
   rescue Errno::ECONNREFUSED, JIRA::HTTPError => e
-    Rails.logger.info "#{self.class.name} Test ERROR: #{url} - #{e.message}"
+    Rails.logger.info "#{self.class.name} ERROR: #{e.message}. API URL: #{url}."
     false
   end
   private
-  def set_jira_issue_transition_id
-    self.jira_issue_transition_id ||= "2"
-  end
-
   def close_issue(entity, issue)
     commit_id = if entity.is_a?(Commit)
                   entity.id
                 elsif entity.is_a?(MergeRequest)
-                  entity.last_commit.id
+                  entity.diff_head_sha
                 end
+
     commit_url = build_entity_url(:commit, commit_id)
     # Depending on the JIRA project's workflow, a comment during transition
@@ -200,57 +206,34 @@ class JiraService < IssueTrackerService
     user_url = data[:user][:url]
     entity_name = data[:entity][:name]
     entity_url = data[:entity][:url]
+    entity_title = data[:entity][:title]
     project_name = data[:project][:name]
-    message = "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]."
-    # unless existing_comment?(issue_name, message[:body])
+    message = "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title}'"
+    unless comment_exists?(issue_key, message)
       send_message(issue_key, message)
-    # end
+    end
   end
-  def send_message(issue_key, message)
-    return unless api_url.present?
-    issue = client.Issue.find(issue_key)
-    issue.comments.build.save!(body: message)
-
-    # message = case result.code
-    #           when 201, 200, 204
-    #             "#{self.class.name} SUCCESS #{result.code}: Successfully posted to #{url}."
-    #           when 401
-    #             "#{self.class.name} ERROR 401: Unauthorized. Check the #{self.username} credentials and JIRA access permissions and try again."
-    #           else
-    #             "#{self.class.name} ERROR #{result.code}: #{result.parsed_response}"
-    #           end
-
-    Rails.logger.info(message)
-    message
-  rescue URI::InvalidURIError, Errno::ECONNREFUSED => e
-    Rails.logger.info "#{self.class.name} Send message ERROR: #{url} - #{e.message}"
+  def comment_exists?(issue_key, message)
+    comments = client.Issue.find(issue_key).comments
+    comments.map { |comment| comment.body.include?(message) }.any?
   end
-  def existing_comment?(issue_name, new_comment)
-    return unless api_url.present?
-    result = JiraService.get(
-      comment_url(issue_name),
-      headers: {
-        'Content-Type' => 'application/json',
-        'Authorization' => "Basic #{auth}"
-      }
-    )
-    case result.code
-    when 201, 200
-      existing_comments = JSON.parse(result.body)['comments']
-      if existing_comments.present?
-        return existing_comments.map { |comment| comment['body'].include?(new_comment) }.any?
-      end
+  def send_message(issue_key, message)
+    return unless url.present?
+    issue = client.Issue.find(issue_key)
+    if issue.comments.build.save!(body: message)
+      result_message = "#{self.class.name} SUCCESS: Successfully posted to #{url}."
     end
-    false
-  rescue JSON::ParserError
-    false
+    Rails.logger.info(result_message)
+    result_message
+  rescue URI::InvalidURIError, Errno::ECONNREFUSED, JIRA::HTTPError => e
+    Rails.logger.info "#{self.class.name} Send message ERROR: #{url} - #{e.message}"
   end
   def resource_url(resource)

app/models/project_services/redmine_service.rb

 class RedmineService < IssueTrackerService
+  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
+
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
   def title

config/gitlab.yml.example

@@ -547,6 +547,10 @@ test:
       project_url: "http://redmine/projects/:issues_tracker_id"
       issues_url: "http://redmine/:project_id/:issues_tracker_id/:id"
       new_issue_url: "http://redmine/projects/:issues_tracker_id/issues/new"
+    jira:
+      title: "JIRA"
+      url: https://sample_company.atlasian.net
+      project_key: PROJECT
   ldap:
     enabled: false
     servers:

db/migrate/20160122231710_migrate_jira_to_gem.rb

-class MigrateJiraToGem < ActiveRecord::Migration
-  def change
-    reversible do |dir|
-      select_all("SELECT id, properties FROM services WHERE services.type IN ('JiraService')").each do |service|
-        id = service['id']
-        properties = JSON.parse(service['properties'])
-        properties_was = properties.clone
-
-        dir.up do
-          # Migrate `project_url` to `project_key`
-          # Ignore if `project_url` doesn't have jql project query with project key
-          if properties['project_url'].present?
-            jql = properties['project_url'].match('project=([A-Za-z]*)')
-            properties['project_key'] = jql.captures.first if jql
-          end
-
-          # Migrate `api_url` to `url`
-          if properties['api_url'].present?
-            url = properties['api_url'].match('(.*)\/rest\/api')
-            properties['url'] = url.captures.first if url
-          end
-
-          # Delete now unnecessary properties
-          properties.delete('api_url')
-          properties.delete('project_url')
-          properties.delete('new_issue_url')
-          properties.delete('issues_url')
-        end
-
-        dir.down do
-          # Rebuild old properties based on sane defaults
-          if properties['url'].present?
-            properties['api_url'] = "#{properties['url']}/rest/api/2"
-            properties['project_url'] =
-              "#{properties['url']}/issues/?jql=project=#{properties['project_key']}"
-            properties['issues_url'] = "#{properties['url']}/browse/:id"
-            properties['new_issue_url'] = "#{properties['url']}/secure/CreateIssue.jspa"
-          end
-
-          # Delete the new properties
-          properties.delete('url')
-          properties.delete('project_key')
-        end
-
-        # Update changes properties
-        if properties != properties_was
-          execute("UPDATE services SET properties = '#{quote_string(properties.to_json)}' WHERE id = #{id}")
-        end
-      end
-    end
-  end
-end

db/migrate/20161011222551_remove_inactive_jira_service_properties.rb

+class RemoveInactiveJiraServiceProperties < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = "Removes all inactive jira_service properties"
+
+  def up
+    execute("UPDATE services SET properties = '{}' WHERE services.type = 'JiraService' and services.active = false")
+  end
+end

db/migrate/20161025231710_migrate_jira_to_gem.rb

+class MigrateJiraToGem < ActiveRecord::Migration
+  DOWNTIME = true
+
+  DOWNTIME_REASON = <<-HEREDOC
+    Refactor all Jira services properties(serialized field) to use new jira-ruby gem.
+    There were properties on old Jira service that are not needed anymore after the
+    service refactoring: api_url, project_url, new_issue_url, issues_url.
+    We extract the new necessary some properties from old keys and delete them:
+    taking project_key from project_url and url from api_url
+  HEREDOC
+
+  def up
+    active_services_query = "SELECT id, properties FROM services WHERE services.type IN ('JiraService') AND services.active = true"
+
+    select_all(active_services_query).each do |service|
+      id = service['id']
+      properties = JSON.parse(service['properties'])
+      properties_was = properties.clone
+
+      # Migrate `project_url` to `project_key`
+      # Ignore if `project_url` doesn't have jql project query with project key
+      if properties['project_url'].present?
+        jql = properties['project_url'].match('project=([A-Za-z]*)')
+        properties['project_key'] = jql.captures.first if jql
+      end
+
+      # Migrate `api_url` to `url`
+      if properties['api_url'].present?
+        url = properties['api_url'].match('(.*)\/rest\/api')
+        properties['url'] = url.captures.first if url
+      end
+
+      # Delete now unnecessary properties
+      properties.delete('api_url')
+      properties.delete('project_url')
+      properties.delete('new_issue_url')
+      properties.delete('issues_url')
+
+      # Update changes properties
+      if properties != properties_was
+        execute("UPDATE services SET properties = '#{quote_string(properties.to_json)}' WHERE id = #{id}")
+      end
+    end
+  end
+
+  def down
+    active_services_query = "SELECT id, properties FROM services WHERE services.type IN ('JiraService') AND services.active = true"
+
+    select_all(active_services_query).each do |service|
+      id = service['id']
+      properties = JSON.parse(service['properties'])
+      properties_was = properties.clone
+
+      # Rebuild old properties based on sane defaults
+      if properties['url'].present?
+        properties['api_url'] = "#{properties['url']}/rest/api/2"
+        properties['project_url'] =
+          "#{properties['url']}/issues/?jql=project=#{properties['project_key']}"
+        properties['issues_url'] = "#{properties['url']}/browse/:id"
+        properties['new_issue_url'] = "#{properties['url']}/secure/CreateIssue.jspa"
+      end
+
+      # Delete the new properties
+      properties.delete('url')
+      properties.delete('project_key')
+
+      # Update changes properties
+      if properties != properties_was
+        execute("UPDATE services SET properties = '#{quote_string(properties.to_json)}' WHERE id = #{id}")
+      end
+    end
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20161024042317) do
+ActiveRecord::Schema.define(version: 20161025231710) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"

doc/integration/README.md

@@ -8,33 +8,54 @@ See the documentation below for details on how to configure these services.
 - [JIRA](jira.md) Integrate with the JIRA issue tracker
 - [External issue tracker](external-issue-tracker.md) Redmine, JIRA, etc.
 - [LDAP](ldap.md) Set up sign in via LDAP
-- [OmniAuth](omniauth.md) Sign in via Twitter, GitHub, GitLab, and Google via OAuth.
+- [OmniAuth](omniauth.md) Sign in via Twitter, GitHub, GitLab.com, Google, Bitbucket, Facebook, Shibboleth, SAML, Crowd and Azure
 - [SAML](saml.md) Configure GitLab as a SAML 2.0 Service Provider
 - [CAS](cas.md) Configure GitLab to sign in using CAS
-- [Slack](slack.md) Integrate with the Slack chat service
 - [OAuth2 provider](oauth_provider.md) OAuth2 application creation
 - [Gmail actions buttons](gmail_action_buttons_for_gitlab.md) Adds GitLab actions to messages
 - [reCAPTCHA](recaptcha.md) Configure GitLab to use Google reCAPTCHA for new users
+- [Akismet](akismet.md) Configure Akismet to stop spam
+- [Koding](../administration/integration/koding.md) Configure Koding to use IDE integration
 GitLab Enterprise Edition contains [advanced Jenkins support][jenkins].
+[jenkins]: http://docs.gitlab.com/ee/integration/jenkins.html
+
+
 ## Project services
 Integration with services such as Campfire, Flowdock, Gemnasium, HipChat,
 Pivotal Tracker, and Slack are available in the form of a [Project Service][].
-You can find these within GitLab in the Services page under Project Settings if
-you are at least a master on the project.
-Project Services are a bit like plugins in that they allow a lot of freedom in
-adding functionality to GitLab. For example there is also a service that can
-send an email every time someone pushes new commits.
-Because GitLab is open source we can ship with the code and tests for all
-plugins. This allows the community to keep the plugins up to date so that they
-always work in newer GitLab versions.
-For an overview of what projects services are available without logging in,
-please see the [project_services directory][projects-code].
-[jenkins]: http://doc.gitlab.com/ee/integration/jenkins.html
-[Project Service]: ../project_services/project_services.md
-[projects-code]: https://gitlab.com/gitlab-org/gitlab-ce/tree/master/app/models/project_services
+[Project Service]: ../project_services/project_services.md
+## SSL certificate errors
+When trying to integrate GitLab with services that are using self-signed certificates,
+it is very likely that SSL certificate errors will occur on different parts of the
+application, most likely Sidekiq. There are 2 approaches you can take to solve this:
+
+1. Add the root certificate to the trusted chain of the OS.
+1. If using Omnibus, you can add the certificate to GitLab's trusted certificates.
+
+**OS main trusted chain**
+
+This [resource](http://kb.kerio.com/product/kerio-connect/server-configuration/ssl-certificates/adding-trusted-root-certificates-to-the-server-1605.html)
+has all the information you need to add a certificate to the main trusted chain.
+
+This [answer](http://superuser.com/questions/437330/how-do-you-add-a-certificate-authority-ca-to-ubuntu)
+at SuperUser also has relevant information.
+
+**Omnibus Trusted Chain**
+
+It is enough to concatenate the certificate to the main trusted certificate:
+
+```bash
+cat jira.pem >> /opt/gitlab/embedded/ssl/certs/cacert.pem
+```
+
+After that restart GitLab with:
+
+```bash
+sudo gitlab-ctl restart
+```