Merge branch 'dm-dependency-linker-gemfile' into 'master'

Autolink package names in Gemfile

See merge request !11224

app/assets/stylesheets/highlight/dark.scss

@@ -185,6 +185,11 @@ $dark-il: #de935f;
     color: $dark-highlight-color !important;
   }
+  // Links to URLs, emails, or dependencies
+  .line a {
+    color: $dark-na;
+  }
+
   .hll { background-color: $dark-hll-bg; }
   .c { color: $dark-c; } /* Comment */
   .err { color: $dark-err; } /* Error */

app/assets/stylesheets/highlight/monokai.scss

@@ -185,6 +185,11 @@ $monokai-gi: #a6e22e;
     color: $black !important;
   }
+  // Links to URLs, emails, or dependencies
+  .line a {
+    color: $monokai-k;
+  }
+
   .hll { background-color: $monokai-hll; }
   .c { color: $monokai-c; } /* Comment */
   .err { color: $monokai-err-color; background-color: $monokai-err-bg; } /* Error */

app/assets/stylesheets/highlight/solarized_dark.scss

@@ -188,6 +188,11 @@ $solarized-dark-il: #2aa198;
     background-color: $solarized-dark-highlight !important;
   }
+  // Links to URLs, emails, or dependencies
+  .line a {
+    color: $solarized-dark-kd;
+  }
+
   /* Solarized Dark
   For use with Jekyll and Pygments

app/assets/stylesheets/highlight/solarized_light.scss

@@ -196,6 +196,11 @@ $solarized-light-il: #2aa198;
     background-color: $solarized-light-highlight !important;
   }
+  // Links to URLs, emails, or dependencies
+  .line a {
+    color: $solarized-light-kd;
+  }
+
   /* Solarized Light
   For use with Jekyll and Pygments

app/assets/stylesheets/highlight/white.scss

@@ -203,6 +203,11 @@ $white-gc-bg: #eaf2f5;
     background-color: $white-highlight !important;
   }
+  // Links to URLs, emails, or dependencies
+  .line a {
+    color: $white-nb;
+  }
+
   .hll { background-color: $white-hll-bg; }
   .c { color: $white-c; font-style: italic; }
   .err { color: $white-err; background-color: $white-err-bg; }

app/services/system_note_service.rb

@@ -291,8 +291,8 @@ module SystemNoteService
     old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new(old_title, new_title).inline_diffs
-    marked_old_title = Gitlab::Diff::InlineDiffMarker.new(old_title).mark(old_diffs, mode: :deletion, markdown: true)
-    marked_new_title = Gitlab::Diff::InlineDiffMarker.new(new_title).mark(new_diffs, mode: :addition, markdown: true)
+    marked_old_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(old_title).mark(old_diffs, mode: :deletion)
+    marked_new_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(new_title).mark(new_diffs, mode: :addition)
     body = "changed title from **#{marked_old_title}** to **#{marked_new_title}**"

changelogs/unreleased/dm-dependency-linker-gemfile.yml

+---
+title: Autolink package names in Gemfile
+merge_request:
+author:

lib/gitlab/dependency_linker.rb

+module Gitlab
+  module DependencyLinker
+    LINKERS = [
+      GemfileLinker,
+    ].freeze
+
+    def self.linker(blob_name)
+      LINKERS.find { |linker| linker.support?(blob_name) }
+    end
+
+    def self.link(blob_name, plain_text, highlighted_text)
+      linker = linker(blob_name)
+      return highlighted_text unless linker
+
+      linker.link(plain_text, highlighted_text)
+    end
+  end
+end

lib/gitlab/dependency_linker/base_linker.rb

+module Gitlab
+  module DependencyLinker
+    class BaseLinker
+      def self.link(plain_text, highlighted_text)
+        new(plain_text, highlighted_text).link
+      end
+
+      attr_accessor :plain_text, :highlighted_text
+
+      def initialize(plain_text, highlighted_text)
+        @plain_text = plain_text
+        @highlighted_text = highlighted_text
+      end
+
+      def link
+        link_dependencies
+
+        highlighted_lines.join.html_safe
+      end
+
+      private
+
+      def package_url(name)
+        raise NotImplementedError
+      end
+
+      def link_dependencies
+        raise NotImplementedError
+      end
+
+      def package_link(name, url = package_url(name))
+        return name unless url
+
+        %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}
+      end
+
+      # Links package names in a method call or assignment string argument.
+      #
+      # Example:
+      #   link_method_call("gem")
+      #   # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"`
+      #
+      #   link_method_call("gem", "specific_package")
+      #   # Will link `specific_package` in `gem "specific_package"`
+      #
+      #   link_method_call("github", /[^\/]+\/[^\/]+/)
+      #   # Will link `user/repo` in `github "user/repo"`, but not `github "package"`
+      #
+      #   link_method_call(%w[add_dependency add_development_dependency])
+      #   # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"`
+      #
+      #   link_method_call("name")
+      #   # Will link `package` in `self.name = "package"`
+      def link_method_call(method_names, value = nil, &url_proc)
+        value =
+          case value
+          when String
+            Regexp.escape(value)
+          when nil
+            /[^'"]+/
+          else
+            value
+          end
+
+        method_names = Array(method_names).map { |name| Regexp.escape(name) }
+
+        regex = %r{
+          #{Regexp.union(method_names)} # Method name
+          \s*                           # Whitespace
+          [(=]?                         # Opening brace or equals sign
+          \s*                           # Whitespace
+          ['"](?<name>#{value})['"]     # Package name in quotes
+        }x
+
+        link_regex(regex, &url_proc)
+      end
+
+      # Links package names based on regex.
+      #
+      # Example:
+      #   link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/)
+      #   # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"`
+      def link_regex(regex)
+        highlighted_lines.map!.with_index do |rich_line, i|
+          marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe)
+
+          marker.mark(regex, group: :name) do |text, left:, right:|
+            url = block_given? ? yield(text) : package_url(text)
+            package_link(text, url)
+          end
+        end
+      end
+
+      def plain_lines
+        @plain_lines ||= plain_text.lines
+      end
+
+      def highlighted_lines
+        @highlighted_lines ||= highlighted_text.lines
+      end
+    end
+  end
+end

lib/gitlab/dependency_linker/gemfile_linker.rb

+module Gitlab
+  module DependencyLinker
+    class GemfileLinker < BaseLinker
+      def self.support?(blob_name)
+        blob_name == 'Gemfile' || blob_name == 'gems.rb'
+      end
+
+      private
+
+      def link_dependencies
+        # Link `gem "package_name"` to https://rubygems.org/gems/package_name
+        link_method_call("gem")
+
+        # Link `github: "user/repo"` to https://github.com/user/repo
+        link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name|
+          "https://github.com/#{name}"
+        end
+
+        # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo
+        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url }
+
+        # Link `source "https://rubygems.org"` to https://rubygems.org
+        link_method_call("source", %r{https?://[^'"]+}) { |url| url }
+      end
+
+      def package_url(name)
+        "https://rubygems.org/gems/#{name}"
+      end
+    end
+  end
+end

lib/gitlab/diff/inline_diff_markdown_marker.rb

+module Gitlab
+  module Diff
+    class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker
+      MARKDOWN_SYMBOLS = {
+        addition: "+",
+        deletion: "-"
+      }.freeze
+
+      def mark(line_inline_diffs, mode: nil)
+        super(line_inline_diffs) do |text, left:, right:|
+          symbol = MARKDOWN_SYMBOLS[mode]
+          "{#{symbol}#{text}#{symbol}}"
+        end
+      end
+    end
+  end
+end

lib/gitlab/diff/inline_diff_marker.rb

 module Gitlab
   module Diff
-    class InlineDiffMarker
-      MARKDOWN_SYMBOLS = {
-        addition: "+",
-        deletion: "-"
-      }.freeze
-
-      attr_accessor :raw_line, :rich_line
-
-      def initialize(raw_line, rich_line = raw_line)
-        @raw_line = raw_line
-        @rich_line = ERB::Util.html_escape(rich_line)
-      end
-
-      def mark(line_inline_diffs, mode: nil, markdown: false)
-        return rich_line unless line_inline_diffs
-
-        marker_ranges = []
-        line_inline_diffs.each do |inline_diff_range|
-          # Map the inline-diff range based on the raw line to character positions in the rich line
-          inline_diff_positions = position_mapping[inline_diff_range].flatten
-          # Turn the array of character positions into ranges
-          marker_ranges.concat(collapse_ranges(inline_diff_positions))
-        end
-
-        offset = 0
-
-        # Mark each range
-        marker_ranges.each_with_index do |range, index|
-          before_content =
-            if markdown
-              "{#{MARKDOWN_SYMBOLS[mode]}"
-            else
-              "<span class='#{html_class_names(marker_ranges, mode, index)}'>"
-            end
-          after_content =
-            if markdown
-              "#{MARKDOWN_SYMBOLS[mode]}}"
-            else
-              "</span>"
-            end
-          offset = insert_around_range(rich_line, range, before_content, after_content, offset)
+    class InlineDiffMarker < Gitlab::StringRangeMarker
+      def mark(line_inline_diffs, mode: nil)
+        super(line_inline_diffs) do |text, left:, right:|
+          %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}
         end
-
-        rich_line.html_safe
       end
       private
-      def html_class_names(marker_ranges, mode, index)
+      def html_class_names(left, right, mode)
         class_names = ["idiff"]
-        class_names << "left"  if index == 0
-        class_names << "right" if index == marker_ranges.length - 1
+        class_names << "left"  if left
+        class_names << "right" if right
         class_names << mode if mode
         class_names.join(" ")
       end
-
-      # Mapping of character positions in the raw line, to the rich (highlighted) line
-      def position_mapping
-        @position_mapping ||= begin
-          mapping = []
-          rich_pos = 0
-          (0..raw_line.length).each do |raw_pos|
-            rich_char = rich_line[rich_pos]
-
-            # The raw and rich lines are the same except for HTML tags,
-            # so skip over any `<...>` segment
-            while rich_char == '<'
-              until rich_char == '>'
-                rich_pos += 1
-                rich_char = rich_line[rich_pos]
-              end
-
-              rich_pos += 1
-              rich_char = rich_line[rich_pos]
-            end
-
-            # multi-char HTML entities in the rich line correspond to a single character in the raw line
-            if rich_char == '&'
-              multichar_mapping = [rich_pos]
-              until rich_char == ';'
-                rich_pos += 1
-                multichar_mapping << rich_pos
-                rich_char = rich_line[rich_pos]
-              end
-
-              mapping[raw_pos] = multichar_mapping
-            else
-              mapping[raw_pos] = rich_pos
-            end
-
-            rich_pos += 1
-          end
-
-          mapping
-        end
-      end
-
-      # Takes an array of integers, and returns an array of ranges covering the same integers
-      def collapse_ranges(positions)
-        return [] if positions.empty?
-        ranges = []
-
-        start = prev = positions[0]
-        range = start..prev
-        positions[1..-1].each do |pos|
-          if pos == prev + 1
-            range = start..pos
-            prev = pos
-          else
-            ranges << range
-            start = prev = pos
-            range = start..prev
-          end
-        end
-        ranges << range
-
-        ranges
-      end
-
-      # Inserts tags around the characters identified by the given range
-      def insert_around_range(text, range, before, after, offset = 0)
-        # Just to be sure
-        return offset if offset + range.end + 1 > text.length
-
-        text.insert(offset + range.begin, before)
-        offset += before.length
-
-        text.insert(offset + range.end + 1, after)
-        offset += after.length
-
-        offset
-      end
     end
   end
 end

lib/gitlab/highlight.rb

@@ -13,6 +13,8 @@ module Gitlab
       highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe)
     end
+    attr_reader :blob_name
+
     def initialize(blob_name, blob_content, repository: nil)
       @formatter = Rouge::Formatters::HTMLGitlab
       @repository = repository
@@ -21,16 +23,9 @@ module Gitlab
     end
     def highlight(text, continue: true, plain: false)
-      if plain
-        hl_lexer = Rouge::Lexers::PlainText
-        continue = false
-      else
-        hl_lexer = self.lexer
-      end
-
-      @formatter.format(hl_lexer.lex(text, continue: continue), tag: hl_lexer.tag).html_safe
-    rescue
-      @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
+      highlighted_text = highlight_text(text, continue: continue, plain: plain)
+      highlighted_text = link_dependencies(text, highlighted_text) if blob_name
+      highlighted_text
     end
     def lexer
@@ -50,5 +45,27 @@ module Gitlab
       Rouge::Lexer.find_fancy(language_name)
     end
+
+    def highlight_text(text, continue: true, plain: false)
+      if plain
+        highlight_plain(text)
+      else
+        highlight_rich(text, continue: continue)
+      end
+    end
+
+    def highlight_plain(text)
+      @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
+    end
+
+    def highlight_rich(text, continue: true)
+      @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe
+    rescue
+      highlight_plain(text)
+    end
+
+    def link_dependencies(text, highlighted_text)
+      Gitlab::DependencyLinker.link(blob_name, text, highlighted_text)
+    end
   end
 end

lib/gitlab/string_range_marker.rb

+module Gitlab
+  class StringRangeMarker
+    attr_accessor :raw_line, :rich_line
+
+    def initialize(raw_line, rich_line = raw_line)
+      @raw_line = raw_line
+      @rich_line = ERB::Util.html_escape(rich_line)
+    end
+
+    def mark(marker_ranges)
+      return rich_line unless marker_ranges
+
+      rich_marker_ranges = []
+      marker_ranges.each do |range|
+        # Map the inline-diff range based on the raw line to character positions in the rich line
+        rich_positions = position_mapping[range].flatten
+        # Turn the array of character positions into ranges
+        rich_marker_ranges.concat(collapse_ranges(rich_positions))
+      end
+
+      offset = 0
+      # Mark each range
+      rich_marker_ranges.each_with_index do |range, i|
+        offset_range = (range.begin + offset)..(range.end + offset)
+        original_text = rich_line[offset_range]
+
+        text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1)
+
+        rich_line[offset_range] = text
+
+        offset += text.length - original_text.length
+      end
+
+      rich_line.html_safe
+    end
+
+    private
+
+    # Mapping of character positions in the raw line, to the rich (highlighted) line
+    def position_mapping
+      @position_mapping ||= begin
+        mapping = []
+        rich_pos = 0
+        (0..raw_line.length).each do |raw_pos|
+          rich_char = rich_line[rich_pos]
+
+          # The raw and rich lines are the same except for HTML tags,
+          # so skip over any `<...>` segment
+          while rich_char == '<'
+            until rich_char == '>'
+              rich_pos += 1
+              rich_char = rich_line[rich_pos]
+            end
+
+            rich_pos += 1
+            rich_char = rich_line[rich_pos]
+          end
+
+          # multi-char HTML entities in the rich line correspond to a single character in the raw line
+          if rich_char == '&'
+            multichar_mapping = [rich_pos]
+            until rich_char == ';'
+              rich_pos += 1
+              multichar_mapping << rich_pos
+              rich_char = rich_line[rich_pos]
+            end
+
+            mapping[raw_pos] = multichar_mapping
+          else
+            mapping[raw_pos] = rich_pos
+          end
+
+          rich_pos += 1
+        end
+
+        mapping
+      end
+    end
+
+    # Takes an array of integers, and returns an array of ranges covering the same integers
+    def collapse_ranges(positions)
+      return [] if positions.empty?
+      ranges = []
+
+      start = prev = positions[0]
+      range = start..prev
+      positions[1..-1].each do |pos|
+        if pos == prev + 1
+          range = start..pos
+          prev = pos
+        else
+          ranges << range
+          start = prev = pos
+          range = start..prev
+        end
+      end
+      ranges << range
+
+      ranges
+    end
+  end
+end

lib/gitlab/string_regex_marker.rb

+module Gitlab
+  class StringRegexMarker < StringRangeMarker
+    def mark(regex, group: 0, &block)
+      regex_match = raw_line.match(regex)
+      return rich_line unless regex_match
+
+      begin_index, end_index = regex_match.offset(group)
+      name_range = begin_index..(end_index - 1)
+
+      super([name_range], &block)
+    end
+  end
+end

spec/helpers/diff_helper_spec.rb

@@ -122,9 +122,9 @@ describe DiffHelper do
     it "returns strings with marked inline diffs" do
       marked_old_line, marked_new_line = mark_inline_diffs(old_line, new_line)
-      expect(marked_old_line).to eq("abc <span class='idiff left right deletion'>&#39;def&#39;</span>")
+      expect(marked_old_line).to eq(%q{abc <span class="idiff left right deletion">&#39;def&#39;</span>})
       expect(marked_old_line).to be_html_safe
-      expect(marked_new_line).to eq("abc <span class='idiff left right addition'>&quot;def&quot;</span>")
+      expect(marked_new_line).to eq(%q{abc <span class="idiff left right addition">&quot;def&quot;</span>})
       expect(marked_new_line).to be_html_safe
     end
   end

spec/lib/gitlab/dependency_linker/gemfile_linker_spec.rb

+require 'rails_helper'
+
+describe Gitlab::DependencyLinker::GemfileLinker, lib: true do
+  describe '.support?' do
+    it 'supports Gemfile' do
+      expect(described_class.support?('Gemfile')).to be_truthy
+    end
+
+    it 'supports gems.rb' do
+      expect(described_class.support?('gems.rb')).to be_truthy
+    end
+
+    it 'does not support other files' do
+      expect(described_class.support?('Gemfile.lock')).to be_falsey
+    end
+  end
+
+  describe '#link' do
+    let(:file_name) { 'Gemfile' }
+
+    let(:file_content) do
+      <<-CONTENT.strip_heredoc
+        source 'https://rubygems.org'
+
+        gem "rails", '4.2.6', github: "rails/rails"
+        gem 'rails-deprecated_sanitizer', '~> 1.0.3'
+        gem 'responders', '~> 2.0', :github => 'rails/responders'
+        gem 'sprockets', '~> 3.6.0', git: 'https://gitlab.example.com/gems/sprockets'
+        gem 'default_value_for', '~> 3.0.0'
+      CONTENT
+    end
+
+    subject { Gitlab::Highlight.highlight(file_name, file_content) }
+
+    def link(name, url)
+      %{<a href="#{url}" rel="noopener noreferrer" target="_blank">#{name}</a>}
+    end
+
+    it 'links sources' do
+      expect(subject).to include(link('https://rubygems.org', 'https://rubygems.org'))
+    end
+
+    it 'links dependencies' do
+      expect(subject).to include(link('rails', 'https://rubygems.org/gems/rails'))
+      expect(subject).to include(link('rails-deprecated_sanitizer', 'https://rubygems.org/gems/rails-deprecated_sanitizer'))
+      expect(subject).to include(link('responders', 'https://rubygems.org/gems/responders'))
+      expect(subject).to include(link('sprockets', 'https://rubygems.org/gems/sprockets'))
+      expect(subject).to include(link('default_value_for', 'https://rubygems.org/gems/default_value_for'))
+    end
+
+    it 'links GitHub repos' do
+      expect(subject).to include(link('rails/rails', 'https://github.com/rails/rails'))
+      expect(subject).to include(link('rails/responders', 'https://github.com/rails/responders'))
+    end
+
+    it 'links Git repos' do
+      expect(subject).to include(link('https://gitlab.example.com/gems/sprockets', 'https://gitlab.example.com/gems/sprockets'))
+    end
+  end
+end

spec/lib/gitlab/dependency_linker_spec.rb

+require 'rails_helper'
+
+describe Gitlab::DependencyLinker, lib: true do
+  describe '.link' do
+    it 'links using GemfileLinker' do
+      blob_name = 'Gemfile'
+
+      expect(described_class::GemfileLinker).to receive(:link)
+
+      described_class.link(blob_name, nil, nil)
+    end
+  end
+end

spec/lib/gitlab/diff/highlight_spec.rb

@@ -34,7 +34,7 @@ describe Gitlab::Diff::Highlight, lib: true do
       end
       it 'highlights and marks added lines' do
-        code = %Q{+<span id="LC9" class="line" lang="ruby">      <span class="k">raise</span> <span class="no"><span class='idiff left'>RuntimeError</span></span><span class="p"><span class='idiff'>,</span></span><span class='idiff right'> </span><span class="s2">"System commands must be given as an array of strings"</span></span>\n}
+        code = %Q{+<span id="LC9" class="line" lang="ruby">      <span class="k">raise</span> <span class="no"><span class="idiff left">RuntimeError</span></span><span class="p"><span class="idiff">,</span></span><span class="idiff right"> </span><span class="s2">"System commands must be given as an array of strings"</span></span>\n}
         expect(subject[5].text).to eq(code)
       end
@@ -67,7 +67,7 @@ describe Gitlab::Diff::Highlight, lib: true do
       end
       it 'marks added lines' do
-        code = %q{+      raise <span class='idiff left right'>RuntimeError, </span>&quot;System commands must be given as an array of strings&quot;}
+        code = %q{+      raise <span class="idiff left right">RuntimeError, </span>&quot;System commands must be given as an array of strings&quot;}
         expect(subject[5].text).to eq(code)
         expect(subject[5].text).to be_html_safe

spec/lib/gitlab/diff/inline_diff_markdown_marker_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Diff::InlineDiffMarkdownMarker, lib: true do
+  describe '#mark' do
+    let(:raw) { "abc 'def'" }
+    let(:inline_diffs) { [2..5] }
+    let(:subject) { described_class.new(raw).mark(inline_diffs, mode: :deletion) }
+
+    it 'marks the range' do
+      expect(subject).to eq("ab{-c 'd-}ef'")
+      expect(subject).to be_html_safe
+    end
+  end
+end