Merge branch 'fix-references-header-parsing' into 'master'

Fix References header parser for Microsoft Exchange

Closes #26567

See merge request !8620

changelogs/unreleased/fix-references-header-parsing.yml

+---
+title: Fix reply by email without sub-addressing for some clients from
+  Microsoft and Apple
+merge_request: 8620
+author:

lib/gitlab/email/receiver.rb

@@ -35,6 +35,8 @@ module Gitlab
         handler.execute
       end
+      private
+
       def build_mail
         Mail::Message.new(@raw)
       rescue Encoding::UndefinedConversionError,
@@ -54,7 +56,24 @@ module Gitlab
       end
       def key_from_additional_headers(mail)
-        Array(mail.references).find do |mail_id|
+        references = ensure_references_array(mail.references)
+
+        find_key_from_references(references)
+      end
+
+      def ensure_references_array(references)
+        case references
+        when Array
+          references
+        when String
+          # Handle emails from clients which append with commas,
+          # example clients are Microsoft exchange and iOS app
+          Gitlab::IncomingEmail.scan_fallback_references(references)
+        end
+      end
+
+      def find_key_from_references(references)
+        references.find do |mail_id|
           key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id)
           break key if key
         end

lib/gitlab/incoming_email.rb

@@ -4,8 +4,6 @@ module Gitlab
     WILDCARD_PLACEHOLDER = '%{key}'.freeze
     class << self
-      FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze
-
       def enabled?
         config.enabled && config.address
       end
@@ -37,10 +35,14 @@ module Gitlab
       end
       def key_from_fallback_message_id(mail_id)
-        match = mail_id.match(FALLBACK_MESSAGE_ID_REGEX)
-        return unless match
-        match[1]
+        message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/
+        mail_id[message_id_regexp, 1]
+      end
+
+      def scan_fallback_references(references)
+        # It's looking for each <...>
+        references.scan(/(?!<)[^<>]+(?=>)/)
       end
       def config

spec/fixtures/emails/reply_without_subaddressing_and_key_inside_references_with_a_comma.eml

+Return-Path: <jake@adventuretime.ooo>
+Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
+Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400
+Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <reply@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 14:03:48 -0700
+Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
+Date: Thu, 13 Jun 2013 17:03:48 -0400
+From: Jake the Dog <jake@adventuretime.ooo>
+To: reply@appmail.adventuretime.ooo
+Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
+In-Reply-To: <issue_1@localhost>
+References: <issue_1@localhost> <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>,<exchange@microsoft.com>
+Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
+Mime-Version: 1.0
+Content-Type: text/plain;
+ charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+X-Sieve: CMU Sieve 2.2
+X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
+ 13 Jun 2013 14:03:48 -0700 (PDT)
+X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1
+
+I could not disagree more. I am obviously biased but adventure time is the
+greatest show ever created. Everyone should watch it.
+
+- Jake out
+
+
+On Sun, Jun 9, 2013 at 1:39 PM, eviltrout via Discourse Meta
+<reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo> wrote:
+>
+>
+>
+> eviltrout posted in 'Adventure Time Sux' on Discourse Meta:
+>
+> ---
+> hey guys everyone knows adventure time sucks!
+>
+> ---
+> Please visit this link to respond: http://localhost:3000/t/adventure-time-sux/1234/3
+>
+> To unsubscribe from these emails, visit your [user preferences](http://localhost:3000/user_preferences).
+>

spec/lib/gitlab/email/handler/create_note_handler_spec.rb

@@ -174,6 +174,12 @@ describe Gitlab::Email::Handler::CreateNoteHandler, lib: true do
         it_behaves_like 'an email that contains a mail key', 'References'
       end
+
+      context 'mail key is in the References header with a comma' do
+        let(:email_raw) { fixture_file('emails/reply_without_subaddressing_and_key_inside_references_with_a_comma.eml') }
+
+        it_behaves_like 'an email that contains a mail key', 'References'
+      end
     end
   end
 end

spec/lib/gitlab/incoming_email_spec.rb

@@ -90,4 +90,19 @@ describe Gitlab::IncomingEmail, lib: true do
       expect(described_class.key_from_fallback_message_id('reply-key@localhost')).to eq('key')
     end
   end
+
+  context 'self.scan_fallback_references' do
+    let(:references) do
+      '<issue_1@localhost>' +
+        ' <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>' +
+        ',<exchange@microsoft.com>'
+    end
+
+    it 'returns reply key' do
+      expect(described_class.scan_fallback_references(references))
+        .to eq(%w[issue_1@localhost
+                  reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost
+                  exchange@microsoft.com])
+    end
+  end
 end