Support Markdown rendering using multiple projects

This refactors the Markdown pipeline so it supports the rendering of
multiple documents that may belong to different projects. An example of
where this happens is when displaying the event feed of a group. In this
case we retrieve events for all projects in the group. Previously we
would group events per project and render these chunks separately, but
this would result in many SQL queries being executed. By extending the
Markdown pipeline to support this out of the box we can drastically
reduce the number of SQL queries.

To achieve this we introduce a new object to the pipeline:
Banzai::RenderContext. This object simply wraps two other objects: an
optional Project instance, and an optional User instance. On its own
this wouldn't be very helpful, but a RenderContext can also be used to
associate HTML documents with specific Project instances. This work is
done in Banzai::ObjectRenderer and allows us to reuse as many queries
(and results) as possible.

app/controllers/concerns/notes_actions.rb

@@ -41,7 +41,7 @@ module NotesActions
     @note = Notes::CreateService.new(note_project, current_user, create_params).execute
     if @note.is_a?(Note)
-      Notes::RenderService.new(current_user).execute([@note], @project)
+      Notes::RenderService.new(current_user).execute([@note])
     end
     respond_to do |format|
@@ -56,7 +56,7 @@ module NotesActions
     @note = Notes::UpdateService.new(project, current_user, note_params).execute(note)
     if @note.is_a?(Note)
-      Notes::RenderService.new(current_user).execute([@note], @project)
+      Notes::RenderService.new(current_user).execute([@note])
     end
     respond_to do |format|

app/controllers/concerns/renders_notes.rb

@@ -4,7 +4,7 @@ module RendersNotes
     preload_noteable_for_regular_notes(notes)
     preload_max_access_for_authors(notes, @project)
     preload_first_time_contribution_for_authors(noteable, notes)
-    Notes::RenderService.new(current_user).execute(notes, @project)
+    Notes::RenderService.new(current_user).execute(notes)
     notes
   end

app/controllers/groups_controller.rb

@@ -173,7 +173,9 @@ class GroupsController < Groups::ApplicationController
       .new(@projects, offset: params[:offset].to_i, filter: event_filter)
       .to_a
-    Events::RenderService.new(current_user).execute(@events, atom_request: request.format.atom?)
+    Events::RenderService
+      .new(current_user)
+      .execute(@events, atom_request: request.format.atom?)
   end
   def user_actions

app/controllers/projects/notes_controller.rb

@@ -68,7 +68,7 @@ class Projects::NotesController < Projects::ApplicationController
   private
   def render_json_with_notes_serializer
-    Notes::RenderService.new(current_user).execute([note], project)
+    Notes::RenderService.new(current_user).execute([note])
     render json: note_serializer.represent(note)
   end

app/helpers/markup_helper.rb

@@ -256,7 +256,7 @@ module MarkupHelper
     return '' unless html.present?
     context.merge!(
-      current_user:   (current_user if defined?(current_user)),
+      current_user: (current_user if defined?(current_user)),
       # RelativeLinkFilter
       commit:         @commit,

app/services/events/render_service.rb

 module Events
   class RenderService < BaseRenderer
     def execute(events, atom_request: false)
-      events.map(&:note).compact.group_by(&:project).each do |project, notes|
-        render_notes(notes, project, atom_request)
-      end
+      notes = events.map(&:note).compact
+
+      render_notes(notes, atom_request)
     end
     private
-    def render_notes(notes, project, atom_request)
-      Notes::RenderService.new(current_user).execute(notes, project, render_options(atom_request))
+    def render_notes(notes, atom_request)
+      Notes::RenderService
+        .new(current_user)
+        .execute(notes, render_options(atom_request))
     end
     def render_options(atom_request)

app/services/notes/render_service.rb

@@ -3,19 +3,18 @@ module Notes
     # Renders a collection of Note instances.
     #
     # notes - The notes to render.
-    # project - The project to use for redacting.
-    # user - The user viewing the notes.
-
+    #
     # Possible options:
+    #
     # requested_path - The request path.
     # project_wiki - The project's wiki.
     # ref - The current Git reference.
     # only_path - flag to turn relative paths into absolute ones.
     # xhtml - flag to save the html in XHTML
-    def execute(notes, project, **opts)
-      renderer = Banzai::ObjectRenderer.new(project, current_user, **opts)
-
-      renderer.render(notes, :note)
+    def execute(notes, options = {})
+      Banzai::ObjectRenderer
+        .new(user: current_user, redaction_context: options)
+        .render(notes, :note)
     end
   end
 end

changelogs/unreleased/rendering-markdown-multiple-projects.yml

+---
+title: Support Markdown rendering using multiple projects
+merge_request:
+author:
+type: performance

lib/banzai/commit_renderer.rb

@@ -3,7 +3,7 @@ module Banzai
     ATTRIBUTES = [:description, :title].freeze
     def self.render(commits, project, user = nil)
-      obj_renderer = ObjectRenderer.new(project, user)
+      obj_renderer = ObjectRenderer.new(user: user, default_project: project)
       ATTRIBUTES.each { |attr| obj_renderer.render(commits, attr) }
     end

lib/banzai/filter/issuable_state_filter.rb

@@ -11,7 +11,8 @@ module Banzai
       def call
         return doc unless context[:issuable_state_filter_enabled]
-        extractor = Banzai::IssuableExtractor.new(project, current_user)
+        context = RenderContext.new(project, current_user)
+        extractor = Banzai::IssuableExtractor.new(context)
         issuables = extractor.extract([doc])
         issuables.each do |node, issuable|

lib/banzai/filter/redactor_filter.rb

@@ -7,7 +7,11 @@ module Banzai
     #
     class RedactorFilter < HTML::Pipeline::Filter
       def call
-        Redactor.new(project, current_user).redact([doc]) unless context[:skip_redaction]
+        unless context[:skip_redaction]
+          context = RenderContext.new(project, current_user)
+
+          Redactor.new(context).redact([doc])
+        end
         doc
       end

lib/banzai/issuable_extractor.rb

@@ -12,11 +12,11 @@ module Banzai
       [@data-reference-type="issue" or @data-reference-type="merge_request"]
     ).freeze
-    attr_reader :project, :user
-    def initialize(project, user)
-      @project = project
-      @user = user
+    attr_reader :context
+    # context - An instance of Banzai::RenderContext.
+    def initialize(context)
+      @context = context
     end
     # Returns Hash in the form { node => issuable_instance }
@@ -25,8 +25,10 @@ module Banzai
         document.xpath(QUERY)
       end
-      issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user)
-      merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user)
+      issue_parser = Banzai::ReferenceParser::IssueParser.new(context)
+
+      merge_request_parser =
+        Banzai::ReferenceParser::MergeRequestParser.new(context)
       issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge(
         merge_request_parser.records_for_nodes(nodes)

lib/banzai/object_renderer.rb

@@ -13,14 +13,13 @@ module Banzai
   # As an example, rendering the attribute `note` would place the unredacted
   # HTML into `note_html` and the redacted HTML into `redacted_note_html`.
   class ObjectRenderer
-    attr_reader :project, :user
-    # project - A Project to use for redacting Markdown.
+    attr_reader :context
+    # default_project - A default Project to use for redacting Markdown.
     # user - The user viewing the Markdown/HTML documents, if any.
     # redaction_context - A Hash containing extra attributes to use during redaction
-    def initialize(project, user = nil, redaction_context = {})
-      @project = project
-      @user = user
+    def initialize(default_project: nil, user: nil, redaction_context: {})
+      @context = RenderContext.new(default_project, user)
       @redaction_context = base_context.merge(redaction_context)
     end
@@ -48,17 +47,21 @@ module Banzai
       pipeline = HTML::Pipeline.new([])
       objects.map do |object|
-        pipeline.to_document(Banzai.render_field(object, attribute))
+        document = pipeline.to_document(Banzai.render_field(object, attribute))
+
+        context.associate_document(document, object)
+
+        document
       end
     end
     def post_process_documents(documents, objects, attribute)
       # Called here to populate cache, refer to IssuableExtractor docs
-      IssuableExtractor.new(project, user).extract(documents)
+      IssuableExtractor.new(context).extract(documents)
       documents.zip(objects).map do |document, object|
-        context = context_for(object, attribute)
-        Banzai::Pipeline[:post_process].to_document(document, context)
+        pipeline_context = context_for(document, object, attribute)
+        Banzai::Pipeline[:post_process].to_document(document, pipeline_context)
       end
     end
@@ -66,20 +69,21 @@ module Banzai
     #
     # Returns an Array containing the redacted documents.
     def redact_documents(documents)
-      redactor = Redactor.new(project, user)
+      redactor = Redactor.new(context)
       redactor.redact(documents)
     end
     # Returns a Banzai context for the given object and attribute.
-    def context_for(object, attribute)
-      @redaction_context.merge(object.banzai_render_context(attribute))
+    def context_for(document, object, attribute)
+      @redaction_context.merge(object.banzai_render_context(attribute)).merge(
+        project: context.project_for_node(document)
+      )
     end
     def base_context
       {
-        current_user: user,
-        project: project,
+        current_user: context.current_user,
         skip_redaction: true
       }
     end

lib/banzai/redactor.rb

@@ -2,13 +2,15 @@ module Banzai
   # Class for removing Markdown references a certain user is not allowed to
   # view.
   class Redactor
-    attr_reader :user, :project
-    # project - A Project to use for redacting links.
-    # user - The currently logged in user (if any).
-    def initialize(project, user = nil)
-      @project = project
-      @user = user
+    attr_reader :context
+    # context - An instance of `Banzai::RenderContext`.
+    def initialize(context)
+      @context = context
+    end
+
+    def user
+      context.current_user
     end
     # Redacts the references in the given Array of documents.
@@ -70,11 +72,11 @@ module Banzai
     end
     def redact_cross_project_references(documents)
-      extractor = Banzai::IssuableExtractor.new(project, user)
+      extractor = Banzai::IssuableExtractor.new(context)
       issuables = extractor.extract(documents)
       issuables.each do |node, issuable|
-        next if issuable.project == project
+        next if issuable.project == context.project_for_node(node)
         node['class'] = node['class'].gsub('has-tooltip', '')
         node['title'] = nil
@@ -95,7 +97,7 @@ module Banzai
       end
       per_type.each do |type, nodes|
-        parser = Banzai::ReferenceParser[type].new(project, user)
+        parser = Banzai::ReferenceParser[type].new(context)
         visible.merge(parser.nodes_visible_to_user(user, nodes))
       end

lib/banzai/reference_extractor.rb

@@ -10,8 +10,8 @@ module Banzai
     end
     def references(type, project, current_user = nil)
-      processor = Banzai::ReferenceParser[type]
-        .new(project, current_user)
+      context = RenderContext.new(project, current_user)
+      processor = Banzai::ReferenceParser[type].new(context)
       processor.process(html_documents)
     end

lib/banzai/reference_parser/base_parser.rb

@@ -45,9 +45,13 @@ module Banzai
         @data_attribute ||= "data-#{reference_type.to_s.dasherize}"
       end
-      def initialize(project = nil, current_user = nil)
-        @project = project
-        @current_user = current_user
+      # context - An instance of `Banzai::RenderContext`.
+      def initialize(context)
+        @context = context
+      end
+
+      def project_for_node(node)
+        context.project_for_node(node)
       end
       # Returns all the nodes containing references that the user can refer to.
@@ -224,7 +228,11 @@ module Banzai
       private
-      attr_reader :current_user, :project
+      attr_reader :context
+
+      def current_user
+        context.current_user
+      end
       # When a feature is disabled or visible only for
       # team members we should not allow team members

lib/banzai/reference_parser/issue_parser.rb

@@ -5,15 +5,10 @@ module Banzai
       def nodes_visible_to_user(user, nodes)
         issues = records_for_nodes(nodes)
-        issues_to_check = issues.values
-        unless can?(user, :read_cross_project)
-          issues_to_check, cross_project_issues = issues_to_check.partition do |issue|
-            issue.project == project
-          end
-        end
-
-        readable_issues = Ability.issues_readable_by_user(issues_to_check, user).to_set
+        issues_to_check, cross_project_issues = partition_issues(issues, user)
+        readable_issues =
+          Ability.issues_readable_by_user(issues_to_check, user).to_set
         nodes.select do |node|
           issue_in_node = issues[node]
@@ -25,7 +20,7 @@ module Banzai
           # but not the issue.
           if readable_issues.include?(issue_in_node)
             true
-          elsif cross_project_issues&.include?(issue_in_node)
+          elsif cross_project_issues.include?(issue_in_node)
             can_read_reference?(user, issue_in_node)
           else
             false
@@ -33,6 +28,32 @@ module Banzai
         end
       end
+      # issues - A Hash mapping HTML nodes to their corresponding Issue
+      #          instances.
+      # user - The current User.
+      def partition_issues(issues, user)
+        return [issues.values, []] if can?(user, :read_cross_project)
+
+        issues_to_check = []
+        cross_project_issues = []
+
+        # We manually partition the data since our input is a Hash and our
+        # output has to be an Array of issues; not an Array of (node, issue)
+        # pairs.
+        issues.each do |node, issue|
+          target =
+            if issue.project == project_for_node(node)
+              issues_to_check
+            else
+              cross_project_issues
+            end
+
+          target << issue
+        end
+
+        [issues_to_check, cross_project_issues]
+      end
+
       def records_for_nodes(nodes)
         @issues_for_nodes ||= grouped_objects_for_nodes(
           nodes,

lib/banzai/reference_parser/user_parser.rb

@@ -58,7 +58,7 @@ module Banzai
       def can_read_project_reference?(node)
         node_id = node.attr('data-project').to_i
-        project && project.id == node_id
+        project_for_node(node)&.id == node_id
       end
       def nodes_user_can_reference(current_user, nodes)
@@ -71,6 +71,7 @@ module Banzai
         nodes.select do |node|
           project_id = node.attr(project_attr)
           user_id = node.attr(author_attr)
+          project = project_for_node(node)
           if project && project_id && project.id == project_id.to_i
             true

lib/banzai/render_context.rb

+# frozen_string_literal: true
+
+module Banzai
+  # Object storing the current user, project, and other details used when
+  # parsing Markdown references.
+  class RenderContext
+    attr_reader :current_user
+
+    # default_project - The default project to use for all documents, if any.
+    # current_user - The user viewing the document, if any.
+    def initialize(default_project = nil, current_user = nil)
+      @current_user = current_user
+      @projects = Hash.new(default_project)
+    end
+
+    # Associates an HTML document with a Project.
+    #
+    # document - The HTML document to map to a Project.
+    # object - The object that produced the HTML document.
+    def associate_document(document, object)
+      # XML nodes respond to "document" but will return a Document instance,
+      # even when they belong to a DocumentFragment.
+      document = document.document if document.fragment?
+
+      @projects[document] = object.project if object.respond_to?(:project)
+    end
+
+    def project_for_node(node)
+      @projects[node.document]
+    end
+  end
+end

spec/lib/banzai/commit_renderer_spec.rb

@@ -6,7 +6,10 @@ describe Banzai::CommitRenderer do
       user = build(:user)
       project = create(:project, :repository)
-      expect(Banzai::ObjectRenderer).to receive(:new).with(project, user).and_call_original
+      expect(Banzai::ObjectRenderer)
+        .to receive(:new)
+        .with(user: user, default_project: project)
+        .and_call_original
       described_class::ATTRIBUTES.each do |attr|
         expect_any_instance_of(Banzai::ObjectRenderer).to receive(:render).with([project.commit], attr).once.and_call_original