Add some specs for GFM AST, minor refactorings

lib/gitlab/gfm/ast/lexer.rb

@@ -84,6 +84,16 @@ module Gitlab
             end
           end
         end
+
+        ##
+        # Processes single token, and returns first lexeme that has been
+        # created.
+        #
+        def self.single(text, token)
+          lexer = new(text, [token])
+          nodes = lexer.process!
+          nodes.first
+        end
       end
     end
   end

lib/gitlab/gfm/ast/parser.rb

@@ -2,16 +2,11 @@ module Gitlab
   module Gfm
     module Ast
       class Parser
-        attr_reader :tree
+        attr_reader :tree, :text
         def initialize(text)
           @text = text
-          @lexer = Lexer.new(@text, [Syntax::Content])
-          @nodes = @lexer.process!
-        end
-
-        def tree
-          @nodes.first
+          @tree = Lexer.single(text, Syntax::Content)
         end
         def recreate

lib/gitlab/gfm/ast/syntax/content.rb

@@ -6,7 +6,7 @@ module Gitlab
         # Main GFM content
         #
         class Content < Node
-          def self.allowed
+          def allowed
             [Syntax::Markdown::CodeBlock, Syntax::Text]
           end
@@ -14,6 +14,10 @@ module Gitlab
             /(?<value>.+)/m
           end
+          def value
+            @text
+          end
+
           def to_s
             nodes.map(&:to_s).join
           end

lib/gitlab/gfm/ast/syntax/markdown/code_block.rb

@@ -4,16 +4,20 @@ module Gitlab
       module Syntax
         module Markdown
           class CodeBlock < Node
+            def allowed
+              []
+            end
+
             def to_s
-              @match[:start_token] + @value + @match[:end_token]
+              @text
             end
-            def lang
-              @match[:lang]
+            def value
+              @text
             end
-            def self.allowed
-              []
+            def lang
+              @match[:lang]
             end
             def self.pattern

lib/gitlab/gfm/ast/syntax/node.rb

@@ -3,7 +3,7 @@ module Gitlab
     module Ast
       module Syntax
         class Node
-          attr_reader :text, :range, :parent, :value, :nodes
+          attr_reader :text, :range, :parent, :nodes
           def initialize(text, range, match, parent)
             @text = text
@@ -16,14 +16,12 @@ module Gitlab
           end
           ##
-          # Process children nodes
+          # Nodes allowed inside this one.
           #
-          def process!
-            @nodes = lexer.new(@text, self.class.allowed, self).process!
-          end
-
-          def index
-            @range.begin
+          # This is pipeline of lexemes, order is relevant.
+          #
+          def allowed
+            raise NotImplementedError
           end
           ##
@@ -34,14 +32,35 @@ module Gitlab
           end
           ##
-          # Is this node a leaf node?
+          # Returns the value of this nodes, without node-specific tokens.
+          #
+          def value
+            raise NotImplementedError
+          end
+
+          ##
+          # Process children nodes
+          #
+          def process!
+            @nodes = lexer.new(value, allowed, self).process!
+          end
+
+          ##
+          # Position of this node in parent
+          #
+          def index
+            @range.begin
+          end
+
+          ##
+          # Returns true if node is a leaf in the three.
           #
           def leaf?
             @nodes.empty?
           end
           ##
-          # Lexer for this node
+          # Each node can have it's own lexer.
           #
           def lexer
             Ast::Lexer
@@ -65,18 +84,7 @@ module Gitlab
           end
           ##
-          # Nodes allowed inside this one.
-          #
-          # This is pipeline of lexemes, order is relevant.
-          #
-          def self.allowed
-            raise NotImplementedError
-          end
-
-          ##
-          # Regexp pattern for this node
-          #
-          # Each pattern must contain at least `value` capture group.
+          # Regexp pattern for this token.
           #
           def self.pattern
             raise NotImplementedError

lib/gitlab/gfm/ast/syntax/text.rb

@@ -6,12 +6,16 @@ module Gitlab
         # Text description
         #
         class Text < Node
-          def to_s
+          def allowed
+            []
+          end
+
+          def value
             @text
           end
-          def self.allowed
-            []
+          def to_s
+            @text
           end
           def self.pattern

spec/lib/gitlab/gfm/ast/syntax/content_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Gfm::Ast::Syntax::Content do
+  describe 'token' do
+    let(:text) { "some multi\n\nline text" }
+
+    it 'matches entire text' do
+      expect(text).to match(described_class.pattern)
+    end
+  end
+
+  describe 'lexeme' do
+    let(:text) { "some text with ```ruby\nblock\n```" }
+    let(:lexeme) { Gitlab::Gfm::Ast::Lexer.single(text, described_class) }
+
+    describe '#nodes' do
+      let(:nodes) { lexeme.nodes }
+
+      it 'correctly instantiates children nodes' do
+        expect(nodes.count).to eq 2
+      end
+    end
+
+    describe '#to_s' do
+      subject { lexeme.to_s }
+      it { is_expected.to eq text }
+    end
+  end
+end

spec/lib/gitlab/gfm/ast/syntax/markdown/code_block_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Gfm::Ast::Syntax::Markdown::CodeBlock do
+  let(:text) { "```ruby\ncode block\n```" }
+
+  describe 'token' do
+    it 'matches entire text' do
+      expect(text).to match described_class.pattern
+    end
+  end
+
+  describe 'lexeme' do
+    let(:lexeme) { Gitlab::Gfm::Ast::Lexer.single(text, described_class) }
+
+    describe '#nodes' do
+      subject { lexeme.nodes }
+      it { is_expected.to be_empty }
+    end
+
+    describe '#leaf?' do
+      subject { lexeme.leaf? }
+      it { is_expected.to be true }
+    end
+
+    describe '#to_s' do
+      subject { lexeme.to_s }
+      it { is_expected.to eq text }
+    end
+
+    describe '#lang' do
+      subject { lexeme.lang }
+      it { is_expected.to eq 'ruby' }
+    end
+  end
+end

spec/lib/gitlab/gfm/ast/syntax/text_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Gfm::Ast::Syntax::Text do
+  describe 'token' do
+    let(:text) { "some multi\n\nline text" }
+
+    it 'matches entire text' do
+      expect(text).to match described_class.pattern
+    end
+  end
+
+  describe 'lexeme' do
+    let(:text) { "some text with ```ruby\nblock\n```" }
+    let(:lexeme) { Gitlab::Gfm::Ast::Lexer.single(text, described_class) }
+
+    describe '#nodes' do
+      subject { lexeme.nodes }
+      it { is_expected.to be_empty }
+    end
+
+    describe '#leaf?' do
+      subject { lexeme.leaf? }
+      it { is_expected.to be true }
+    end
+
+    describe '#to_s' do
+      subject { lexeme.to_s }
+      it { is_expected.to eq text }
+    end
+  end
+end