Merge branch '22591-Convert-UTF-8-Emoji-to-Gitlab-emoji' into 'master'

Convert unicode emojis to images.

## Why was this MR needed?

For better cross platform interoperability with emojis.

Closes #22591

See merge request !6829

CHANGELOG

@@ -38,6 +38,7 @@ v 8.13.0 (unreleased)
   - API: Ability to retrieve version information (Robert Schilling)
   - Fix permission for setting an issue's due date
   - API: Multi-file commit !6096 (mahcsig)
+  - Unicode emoji are now converted to images
   - Revert "Label list shows all issues (opened or closed) with that label"
   - Expose expires_at field when sharing project on API
   - Fix VueJS template tags being rendered in code comments

lib/banzai/filter/emoji_filter.rb

 module Banzai
   module Filter
-    # HTML filter that replaces :emoji: with images.
+    # HTML filter that replaces :emoji: and unicode with images.
     #
     # Based on HTML::Pipeline::EmojiFilter
     #
@@ -13,16 +13,17 @@ module Banzai
       def call
         search_text_nodes(doc).each do |node|
           content = node.to_html
-          next unless content.include?(':')
           next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS)
-          html = emoji_image_filter(content)
+          next unless content.include?(':') || node.text.match(emoji_unicode_pattern)
+
+          html = emoji_name_image_filter(content)
+          html = emoji_unicode_image_filter(html)
           next if html == content
           node.replace(html)
         end
-
         doc
       end
@@ -31,18 +32,38 @@ module Banzai
       # text - String text to replace :emoji: in.
       #
       # Returns a String with :emoji: replaced with images.
-      def emoji_image_filter(text)
+      def emoji_name_image_filter(text)
         text.gsub(emoji_pattern) do |match|
           name = $1
-          "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{emoji_url(name)}' height='20' width='20' align='absmiddle' />"
+          emoji_image_tag(name, emoji_url(name))
         end
       end
+      # Replace unicode emoji with corresponding images if they exist.
+      #
+      # text - String text to replace unicode emoji in.
+      #
+      # Returns a String with unicode emoji replaced with images.
+      def emoji_unicode_image_filter(text)
+        text.gsub(emoji_unicode_pattern) do |moji|
+          emoji_image_tag(Gitlab::Emoji.emojis_by_moji[moji]['name'], emoji_unicode_url(moji))
+        end
+      end
+
+      def emoji_image_tag(emoji_name, emoji_url)
+        "<img class='emoji' title=':#{emoji_name}:' alt=':#{emoji_name}:' src='#{emoji_url}' height='20' width='20' align='absmiddle' />"
+      end
+
       # Build a regexp that matches all valid :emoji: names.
       def self.emoji_pattern
         @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/
       end
+      # Build a regexp that matches all valid unicode emojis names.
+      def self.emoji_unicode_pattern
+        @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/
+      end
+
       private
       def emoji_url(name)
@@ -60,6 +81,18 @@ module Banzai
         end
       end
+      def emoji_unicode_url(moji)
+        emoji_unicode_path = emoji_unicode_filename(moji)
+
+        if context[:asset_host]
+          url_to_image(emoji_unicode_path)
+        elsif context[:asset_root]
+          File.join(context[:asset_root], url_to_image(emoji_unicode_path))
+        else
+          url_to_image(emoji_unicode_path)
+        end
+      end
+
       def url_to_image(image)
         ActionController::Base.helpers.url_to_image(image)
       end
@@ -71,6 +104,14 @@ module Banzai
       def emoji_filename(name)
         "#{Gitlab::Emoji.emoji_filename(name)}.png"
       end
+
+      def emoji_unicode_pattern
+        self.class.emoji_unicode_pattern
+      end
+
+      def emoji_unicode_filename(name)
+        "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png"
+      end
     end
   end
 end

lib/gitlab/emoji.rb

@@ -10,12 +10,20 @@ module Gitlab
       Gemojione.index.instance_variable_get(:@emoji_by_moji)
     end
+    def emojis_unicodes
+      emojis_by_moji.keys
+    end
+
     def emojis_names
-      emojis.keys.sort
+      emojis.keys
     end
     def emoji_filename(name)
       emojis[name]["unicode"]
     end
+
+    def emoji_unicode_filename(moji)
+      emojis_by_moji[moji]["unicode"]
+    end
   end
 end

spec/lib/banzai/filter/emoji_filter_spec.rb

@@ -12,11 +12,16 @@ describe Banzai::Filter::EmojiFilter, lib: true do
     ActionController::Base.asset_host = @original_asset_host
   end
-  it 'replaces supported emoji' do
+  it 'replaces supported name emoji' do
     doc = filter('<p>:heart:</p>')
     expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/2764.png'
   end
+  it 'replaces supported unicode emoji' do
+    doc = filter('<p>❤️</p>')
+    expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/2764.png'
+  end
+
   it 'ignores unsupported emoji' do
     exp = act = '<p>:foo:</p>'
     doc = filter(act)
@@ -28,46 +33,96 @@ describe Banzai::Filter::EmojiFilter, lib: true do
     expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/1F44D.png'
   end
+  it 'correctly encodes unicode to the URL' do
+    doc = filter('<p>👍</p>')
+    expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/1F44D.png'
+  end
+
   it 'matches at the start of a string' do
     doc = filter(':+1:')
     expect(doc.css('img').size).to eq 1
   end
+  it 'unicode matches at the start of a string' do
+    doc = filter("'👍'")
+    expect(doc.css('img').size).to eq 1
+  end
+
   it 'matches at the end of a string' do
     doc = filter('This gets a :-1:')
     expect(doc.css('img').size).to eq 1
   end
+  it 'unicode matches at the end of a string' do
+    doc = filter('This gets a 👍')
+    expect(doc.css('img').size).to eq 1
+  end
+
   it 'matches with adjacent text' do
     doc = filter('+1 (:+1:)')
     expect(doc.css('img').size).to eq 1
   end
+  it 'unicode matches with adjacent text' do
+    doc = filter('+1 (👍)')
+    expect(doc.css('img').size).to eq 1
+  end
+
   it 'matches multiple emoji in a row' do
     doc = filter(':see_no_evil::hear_no_evil::speak_no_evil:')
     expect(doc.css('img').size).to eq 3
   end
+  it 'unicode matches multiple emoji in a row' do
+    doc = filter("'🙈🙉🙊'")
+    expect(doc.css('img').size).to eq 3
+  end
+
+  it 'mixed matches multiple emoji in a row' do
+    doc = filter("'🙈:see_no_evil:🙉:hear_no_evil:🙊:speak_no_evil:'")
+    expect(doc.css('img').size).to eq 6
+  end
+
   it 'has a title attribute' do
     doc = filter(':-1:')
     expect(doc.css('img').first.attr('title')).to eq ':-1:'
   end
+  it 'unicode has a title attribute' do
+    doc = filter("'👎'")
+    expect(doc.css('img').first.attr('title')).to eq ':thumbsdown:'
+  end
+
   it 'has an alt attribute' do
     doc = filter(':-1:')
     expect(doc.css('img').first.attr('alt')).to eq ':-1:'
   end
+  it 'unicode has an alt attribute' do
+    doc = filter("'👎'")
+    expect(doc.css('img').first.attr('alt')).to eq ':thumbsdown:'
+  end
+
   it 'has an align attribute' do
     doc = filter(':8ball:')
     expect(doc.css('img').first.attr('align')).to eq 'absmiddle'
   end
+  it 'unicode has an align attribute' do
+    doc = filter("'🎱'")
+    expect(doc.css('img').first.attr('align')).to eq 'absmiddle'
+  end
+
   it 'has an emoji class' do
     doc = filter(':cat:')
     expect(doc.css('img').first.attr('class')).to eq 'emoji'
   end
+  it 'unicode has an emoji class' do
+    doc = filter("'🐱'")
+    expect(doc.css('img').first.attr('class')).to eq 'emoji'
+  end
+
   it 'has height and width attributes' do
     doc = filter(':dog:')
     img = doc.css('img').first
@@ -76,12 +131,26 @@ describe Banzai::Filter::EmojiFilter, lib: true do
     expect(img.attr('height')).to eq '20'
   end
+  it 'unicode has height and width attributes' do
+    doc = filter("'🐶'")
+    img = doc.css('img').first
+
+    expect(img.attr('width')).to eq '20'
+    expect(img.attr('height')).to eq '20'
+  end
+
   it 'keeps whitespace intact' do
     doc = filter('This deserves a :+1:, big time.')
     expect(doc.to_html).to match(/^This deserves a <img.+>, big time\.\z/)
   end
+  it 'unicode keeps whitespace intact' do
+    doc = filter('This deserves a 🎱, big time.')
+
+    expect(doc.to_html).to match(/^This deserves a <img.+>, big time\.\z/)
+  end
+
   it 'uses a custom asset_root context' do
     root = Gitlab.config.gitlab.url + 'gitlab/root'
@@ -95,4 +164,18 @@ describe Banzai::Filter::EmojiFilter, lib: true do
     doc = filter(':frowning:', asset_host: 'https://this-is-ignored-i-guess?')
     expect(doc.css('img').first.attr('src')).to start_with('https://cdn.example.com')
   end
+
+  it 'uses a custom asset_root context' do
+    root = Gitlab.config.gitlab.url + 'gitlab/root'
+
+    doc = filter("'🎱'", asset_root: root)
+    expect(doc.css('img').first.attr('src')).to start_with(root)
+  end
+
+  it 'uses a custom asset_host context' do
+    ActionController::Base.asset_host = 'https://cdn.example.com'
+
+    doc = filter("'🎱'", asset_host: 'https://this-is-ignored-i-guess?')
+    expect(doc.css('img').first.attr('src')).to start_with('https://cdn.example.com')
+  end
 end