Merge tag 'v0.5.4' of https://github.com/tmm1/pygments.rb

vendor/pygments-main/pygments/formatters/other.py

@@ -5,7 +5,7 @@
     Other formatters: NullFormatter, RawTokenFormatter.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
@@ -102,8 +102,6 @@ class RawTokenFormatter(Formatter):
                 outfile.write(text.encode())
             flush = outfile.flush
-        lasttype = None
-        lastval = u''
         if self.error_color:
             for ttype, value in tokensource:
                 line = "%s\t%r\n" % (ttype, value)

vendor/pygments-main/pygments/formatters/rtf.py

@@ -5,7 +5,7 @@
     A formatter that generates RTF files.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

vendor/pygments-main/pygments/formatters/svg.py

@@ -5,7 +5,7 @@
     Formatter for SVG output.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

vendor/pygments-main/pygments/formatters/terminal.py

@@ -5,7 +5,7 @@
     Formatter for terminal output with ANSI sequences.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

vendor/pygments-main/pygments/formatters/terminal256.py

@@ -11,7 +11,7 @@
     Formatter version 1.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

vendor/pygments-main/pygments/lexer.py

@@ -5,10 +5,10 @@
     Base lexer classes.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-import re
+import re, itertools
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
@@ -18,7 +18,7 @@ from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
-           'LexerContext', 'include', 'bygroups', 'using', 'this']
+           'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this']
 _encoding_map = [('\xef\xbb\xbf', 'utf-8'),
@@ -72,15 +72,18 @@ class Lexer(object):
     #: Shortcuts for the lexer
     aliases = []
-    #: fn match rules
+    #: File name globs
     filenames = []
-    #: fn alias filenames
+    #: Secondary file name globs
     alias_filenames = []
-    #: mime types
+    #: MIME types
     mimetypes = []
+    #: Priority, should multiple lexers match and no content is provided
+    priority = 0
+
     __metaclass__ = LexerMeta
     def __init__(self, **options):
@@ -163,6 +166,10 @@ class Lexer(object):
                 text = decoded
             else:
                 text = text.decode(self.encoding)
+        else:
+            if text.startswith(u'\ufeff'):
+                text = text[len(u'\ufeff'):]
+
         # text now *is* a unicode string
         text = text.replace('\r\n', '\n')
         text = text.replace('\r', '\n')
@@ -238,6 +245,16 @@ class include(str):
     pass
+class _inherit(object):
+    """
+    Indicates the a state should inherit from its superclass.
+    """
+    def __repr__(self):
+        return 'inherit'
+
+inherit = _inherit()
+
+
 class combined(tuple):
     """
     Indicates a state combined from multiple states.
@@ -428,6 +445,9 @@ class RegexLexerMeta(LexerMeta):
                 tokens.extend(cls._process_state(unprocessed, processed,
                                                  str(tdef)))
                 continue
+            if isinstance(tdef, _inherit):
+                # processed already
+                continue
             assert type(tdef) is tuple, "wrong rule def %r" % tdef
@@ -456,6 +476,49 @@ class RegexLexerMeta(LexerMeta):
             cls._process_state(tokendefs, processed, state)
         return processed
+    def get_tokendefs(cls):
+        """
+        Merge tokens from superclasses in MRO order, returning a single tokendef
+        dictionary.
+
+        Any state that is not defined by a subclass will be inherited
+        automatically.  States that *are* defined by subclasses will, by
+        default, override that state in the superclass.  If a subclass wishes to
+        inherit definitions from a superclass, it can use the special value
+        "inherit", which will cause the superclass' state definition to be
+        included at that point in the state.
+        """
+        tokens = {}
+        inheritable = {}
+        for c in itertools.chain((cls,), cls.__mro__):
+            toks = c.__dict__.get('tokens', {})
+
+            for state, items in toks.iteritems():
+                curitems = tokens.get(state)
+                if curitems is None:
+                    tokens[state] = items
+                    try:
+                        inherit_ndx = items.index(inherit)
+                    except ValueError:
+                        continue
+                    inheritable[state] = inherit_ndx
+                    continue
+
+                inherit_ndx = inheritable.pop(state, None)
+                if inherit_ndx is None:
+                    continue
+
+                # Replace the "inherit" value with the items
+                curitems[inherit_ndx:inherit_ndx+1] = items
+                try:
+                    new_inh_ndx = items.index(inherit)
+                except ValueError:
+                    pass
+                else:
+                    inheritable[state] = inherit_ndx + new_inh_ndx
+
+        return tokens
+
     def __call__(cls, *args, **kwds):
         """Instantiate cls after preprocessing its token definitions."""
         if '_tokens' not in cls.__dict__:
@@ -465,7 +528,7 @@ class RegexLexerMeta(LexerMeta):
                 # don't process yet
                 pass
             else:
-                cls._tokens = cls.process_tokendef('', cls.tokens)
+                cls._tokens = cls.process_tokendef('', cls.get_tokendefs())
         return type.__call__(cls, *args, **kwds)
@@ -606,7 +669,13 @@ class ExtendedRegexLexer(RegexLexer):
                     if new_state is not None:
                         # state transition
                         if isinstance(new_state, tuple):
-                            ctx.stack.extend(new_state)
+                            for state in new_state:
+                                if state == '#pop':
+                                    ctx.stack.pop()
+                                elif state == '#push':
+                                    ctx.stack.append(statestack[-1])
+                                else:
+                                    ctx.stack.append(state)
                         elif isinstance(new_state, int):
                             # pop
                             del ctx.stack[new_state:]
@@ -622,10 +691,10 @@ class ExtendedRegexLexer(RegexLexer):
                         break
                     if text[ctx.pos] == '\n':
                         # at EOL, reset state to "root"
-                        ctx.pos += 1
                         ctx.stack = ['root']
                         statetokens = tokendefs['root']
                         yield ctx.pos, Text, u'\n'
+                        ctx.pos += 1
                         continue
                     yield ctx.pos, Error, text[ctx.pos]
                     ctx.pos += 1
@@ -694,4 +763,3 @@ def do_insertions(insertions, tokens):
         except StopIteration:
             insleft = False
             break  # not strictly necessary
-

vendor/pygments-main/pygments/lexers/__init__.py

@@ -5,7 +5,7 @@
     Pygments lexers.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
@@ -15,6 +15,7 @@ import fnmatch
 from os.path import basename
 from pygments.lexers._mapping import LEXERS
+from pygments.modeline import get_filetype_from_buffer
 from pygments.plugin import find_plugin_lexers
 from pygments.util import ClassNotFound, bytes
@@ -113,7 +114,7 @@ def get_lexer_for_filename(_fn, code=None, **options):
         # to find lexers which need it overridden.
         if code:
             return cls.analyse_text(code) + bonus
-        return bonus
+        return cls.priority + bonus
     if matches:
         matches.sort(key=get_rating)
@@ -197,6 +198,16 @@ def guess_lexer(_text, **options):
     """
     Guess a lexer by strong distinctions in the text (eg, shebang).
     """
+
+    # try to get a vim modeline first
+    ft = get_filetype_from_buffer(_text)
+
+    if ft is not None:
+        try:
+            return get_lexer_by_name(ft, **options)
+        except ClassNotFound:
+            pass
+
     best_lexer = [0.0, None]
     for lexer in _iter_lexerclasses():
         rv = lexer.analyse_text(_text)

vendor/pygments-main/pygments/lexers/_asybuiltins.py

@@ -10,7 +10,7 @@
     TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only
     for function and variable names.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

vendor/pygments-main/pygments/lexers/_clbuiltins.py

@@ -5,7 +5,7 @@
     ANSI Common Lisp builtins.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

vendor/pygments-main/pygments/lexers/_luabuiltins.py

@@ -9,7 +9,7 @@
     Do not edit the MODULES dict by hand.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

vendor/pygments-main/pygments/lexers/_phpbuiltins.py

@@ -12,7 +12,7 @@
              internet connection. don't run that at home, use
              a server ;-)
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
@@ -3784,4 +3784,4 @@ if __name__ == '__main__':
         regenerate(__file__, modules)
         shutil.rmtree(PHP_MANUAL_DIR)
-    run()
\ No newline at end of file
+    run()

vendor/pygments-main/pygments/lexers/_postgres_builtins.py

+# -*- coding: utf-8 -*-
 """
     pygments.lexers._postgres_builtins
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Self-updating data files for PostgreSQL lexer.
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
@@ -41,7 +42,6 @@ def parse_keywords(f):
 def parse_datatypes(f):
     dt = set()
-    re_entry = re.compile('\s*<entry><type>([^<]+)</type></entry>')
     for line in f:
         if '<sect1' in line:
             break
@@ -57,7 +57,8 @@ def parse_datatypes(f):
         line = re.sub("<[^>]+>", "", line)
         # Drop the parts containing braces
-        for tmp in [ t for tmp in line.split('[') for t in tmp.split(']') if "(" not in t ]:
+        for tmp in [t for tmp in line.split('[')
+                    for t in tmp.split(']') if "(" not in t]:
             for t in tmp.split(','):
                 t = t.strip()
                 if not t: continue

vendor/pygments-main/pygments/lexers/_robotframeworklexer.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers._robotframeworklexer
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Robot Framework.
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+#  Copyright 2012 Nokia Siemens Networks Oyj
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import re
+
+from pygments.lexer import Lexer
+from pygments.token import Token
+
+
+HEADING = Token.Generic.Heading
+SETTING = Token.Keyword.Namespace
+IMPORT = Token.Name.Namespace
+TC_KW_NAME = Token.Generic.Subheading
+KEYWORD = Token.Name.Function
+ARGUMENT = Token.String
+VARIABLE = Token.Name.Variable
+COMMENT = Token.Comment
+SEPARATOR = Token.Punctuation
+SYNTAX = Token.Punctuation
+GHERKIN = Token.Generic.Emph
+ERROR = Token.Error
+
+
+def normalize(string, remove=''):
+    string = string.lower()
+    for char in remove + ' ':
+        if char in string:
+            string = string.replace(char, '')
+    return string
+
+
+class RobotFrameworkLexer(Lexer):
+    """
+    For `Robot Framework <http://robotframework.org>`_ test data.
+
+    Supports both space and pipe separated plain text formats.
+
+    *New in Pygments 1.6.*
+    """
+    name = 'RobotFramework'
+    aliases = ['RobotFramework', 'robotframework']
+    filenames = ['*.txt', '*.robot']
+    mimetypes = ['text/x-robotframework']
+
+    def __init__(self, **options):
+        options['tabsize'] = 2
+        options['encoding'] = 'UTF-8'
+        Lexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        row_tokenizer = RowTokenizer()
+        var_tokenizer = VariableTokenizer()
+        index = 0
+        for row in text.splitlines():
+            for value, token in row_tokenizer.tokenize(row):
+                for value, token in var_tokenizer.tokenize(value, token):
+                    if value:
+                        yield index, token, unicode(value)
+                        index += len(value)
+
+
+class VariableTokenizer(object):
+
+    def tokenize(self, string, token):
+        var = VariableSplitter(string, identifiers='$@%')
+        if var.start < 0 or token in (COMMENT, ERROR):
+            yield string, token
+            return
+        for value, token in self._tokenize(var, string, token):
+            if value:
+                yield value, token
+
+    def _tokenize(self, var, string, orig_token):
+        before = string[:var.start]
+        yield before, orig_token
+        yield var.identifier + '{', SYNTAX
+        for value, token in self.tokenize(var.base, VARIABLE):
+            yield value, token
+        yield '}', SYNTAX
+        if var.index:
+            yield '[', SYNTAX
+            for value, token in self.tokenize(var.index, VARIABLE):
+                yield value, token
+            yield ']', SYNTAX
+        for value, token in self.tokenize(string[var.end:], orig_token):
+            yield value, token
+
+
+class RowTokenizer(object):
+
+    def __init__(self):
+        self._table = UnknownTable()
+        self._splitter = RowSplitter()
+        testcases = TestCaseTable()
+        settings = SettingTable(testcases.set_default_template)
+        variables = VariableTable()
+        keywords = KeywordTable()
+        self._tables = {'settings': settings, 'setting': settings,
+                        'metadata': settings,
+                        'variables': variables, 'variable': variables,
+                        'testcases': testcases, 'testcase': testcases,
+                        'keywords': keywords, 'keyword': keywords,
+                        'userkeywords': keywords, 'userkeyword': keywords}
+
+    def tokenize(self, row):
+        commented = False
+        heading = False
+        for index, value in enumerate(self._splitter.split(row)):
+            # First value, and every second after that, is a separator.
+            index, separator = divmod(index-1, 2)
+            if value.startswith('#'):
+                commented = True
+            elif index == 0 and value.startswith('*'):
+                self._table = self._start_table(value)
+                heading = True
+            for value, token in self._tokenize(value, index, commented,
+                                               separator, heading):
+                yield value, token
+        self._table.end_row()
+
+    def _start_table(self, header):
+        name = normalize(header, remove='*')
+        return self._tables.get(name, UnknownTable())
+
+    def _tokenize(self, value, index, commented, separator, heading):
+        if commented:
+            yield value, COMMENT
+        elif separator:
+            yield value, SEPARATOR
+        elif heading:
+            yield value, HEADING
+        else:
+            for value, token in self._table.tokenize(value, index):
+                yield value, token
+
+
+class RowSplitter(object):
+    _space_splitter = re.compile('( {2,})')
+    _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))')
+
+    def split(self, row):
+        splitter = (row.startswith('| ') and self._split_from_pipes
+                    or self._split_from_spaces)
+        for value in splitter(row):
+            yield value
+        yield '\n'
+
+    def _split_from_spaces(self, row):
+        yield ''  # Start with (pseudo)separator similarly as with pipes
+        for value in self._space_splitter.split(row):
+            yield value
+
+    def _split_from_pipes(self, row):
+        _, separator, rest = self._pipe_splitter.split(row, 1)
+        yield separator
+        while self._pipe_splitter.search(rest):
+            cell, separator, rest = self._pipe_splitter.split(rest, 1)
+            yield cell
+            yield separator
+        yield rest
+
+
+class Tokenizer(object):
+    _tokens = None
+
+    def __init__(self):
+        self._index = 0
+
+    def tokenize(self, value):
+        values_and_tokens = self._tokenize(value, self._index)
+        self._index += 1
+        if isinstance(values_and_tokens, type(Token)):
+            values_and_tokens = [(value, values_and_tokens)]
+        return values_and_tokens
+
+    def _tokenize(self, value, index):
+        index = min(index, len(self._tokens) - 1)
+        return self._tokens[index]
+
+    def _is_assign(self, value):
+        if value.endswith('='):
+            value = value[:-1].strip()
+        var = VariableSplitter(value, identifiers='$@')
+        return var.start == 0 and var.end == len(value)
+
+
+class Comment(Tokenizer):
+    _tokens = (COMMENT,)
+
+
+class Setting(Tokenizer):
+    _tokens = (SETTING, ARGUMENT)
+    _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown',
+                         'suitepostcondition', 'testsetup', 'testprecondition',
+                         'testteardown', 'testpostcondition', 'testtemplate')
+    _import_settings = ('library', 'resource', 'variables')
+    _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags',
+                       'testtimeout')
+    _custom_tokenizer = None
+
+    def __init__(self, template_setter=None):
+        Tokenizer.__init__(self)
+        self._template_setter = template_setter
+
+    def _tokenize(self, value, index):
+        if index == 1 and self._template_setter:
+            self._template_setter(value)
+        if index == 0:
+            normalized = normalize(value)
+            if normalized in self._keyword_settings:
+                self._custom_tokenizer = KeywordCall(support_assign=False)
+            elif normalized in self._import_settings:
+                self._custom_tokenizer = ImportSetting()
+            elif normalized not in self._other_settings:
+                return ERROR
+        elif self._custom_tokenizer:
+            return self._custom_tokenizer.tokenize(value)
+        return Tokenizer._tokenize(self, value, index)
+
+
+class ImportSetting(Tokenizer):
+    _tokens = (IMPORT, ARGUMENT)
+
+
+class TestCaseSetting(Setting):
+    _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition',
+                         'template')
+    _import_settings = ()
+    _other_settings = ('documentation', 'tags', 'timeout')
+
+    def _tokenize(self, value, index):
+        if index == 0:
+            type = Setting._tokenize(self, value[1:-1], index)
+            return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)]
+        return Setting._tokenize(self, value, index)
+
+
+class KeywordSetting(TestCaseSetting):
+    _keyword_settings = ('teardown',)
+    _other_settings = ('documentation', 'arguments', 'return', 'timeout')
+
+
+class Variable(Tokenizer):
+    _tokens = (SYNTAX, ARGUMENT)
+
+    def _tokenize(self, value, index):
+        if index == 0 and not self._is_assign(value):
+            return ERROR
+        return Tokenizer._tokenize(self, value, index)
+
+
+class KeywordCall(Tokenizer):
+    _tokens = (KEYWORD, ARGUMENT)
+
+    def __init__(self, support_assign=True):
+        Tokenizer.__init__(self)
+        self._keyword_found = not support_assign
+        self._assigns = 0
+
+    def _tokenize(self, value, index):
+        if not self._keyword_found and self._is_assign(value):
+            self._assigns += 1
+            return SYNTAX  # VariableTokenizer tokenizes this later.
+        if self._keyword_found:
+            return Tokenizer._tokenize(self, value, index - self._assigns)
+        self._keyword_found = True
+        return GherkinTokenizer().tokenize(value, KEYWORD)
+
+
+class GherkinTokenizer(object):
+    _gherkin_prefix = re.compile('^(Given|When|Then|And) ', re.IGNORECASE)
+
+    def tokenize(self, value, token):
+        match = self._gherkin_prefix.match(value)
+        if not match:
+            return [(value, token)]
+        end = match.end()
+        return [(value[:end], GHERKIN), (value[end:], token)]
+
+
+class TemplatedKeywordCall(Tokenizer):
+    _tokens = (ARGUMENT,)
+
+
+class ForLoop(Tokenizer):
+
+    def __init__(self):
+        Tokenizer.__init__(self)
+        self._in_arguments = False
+
+    def _tokenize(self, value, index):
+        token = self._in_arguments and ARGUMENT or SYNTAX
+        if value.upper() in ('IN', 'IN RANGE'):
+            self._in_arguments = True
+        return token
+
+
+class _Table(object):
+    _tokenizer_class = None
+
+    def __init__(self, prev_tokenizer=None):
+        self._tokenizer = self._tokenizer_class()
+        self._prev_tokenizer = prev_tokenizer
+        self._prev_values_on_row = []
+
+    def tokenize(self, value, index):
+        if self._continues(value, index):
+            self._tokenizer = self._prev_tokenizer
+            yield value, SYNTAX
+        else:
+            for value_and_token in self._tokenize(value, index):
+                yield value_and_token
+        self._prev_values_on_row.append(value)
+
+    def _continues(self, value, index):
+        return value == '...' and all(self._is_empty(t)
+                                      for t in self._prev_values_on_row)
+
+    def _is_empty(self, value):
+        return value in ('', '\\')
+
+    def _tokenize(self, value, index):
+        return self._tokenizer.tokenize(value)
+
+    def end_row(self):
+        self.__init__(prev_tokenizer=self._tokenizer)
+
+
+class UnknownTable(_Table):
+    _tokenizer_class = Comment
+
+    def _continues(self, value, index):
+        return False
+
+
+class VariableTable(_Table):
+    _tokenizer_class = Variable
+
+
+class SettingTable(_Table):
+    _tokenizer_class = Setting
+
+    def __init__(self, template_setter, prev_tokenizer=None):
+        _Table.__init__(self, prev_tokenizer)
+        self._template_setter = template_setter
+
+    def _tokenize(self, value, index):
+        if index == 0 and normalize(value) == 'testtemplate':
+            self._tokenizer = Setting(self._template_setter)
+        return _Table._tokenize(self, value, index)
+
+    def end_row(self):
+        self.__init__(self._template_setter, prev_tokenizer=self._tokenizer)
+
+
+class TestCaseTable(_Table):
+    _setting_class = TestCaseSetting
+    _test_template = None
+    _default_template = None
+
+    @property
+    def _tokenizer_class(self):
+        if self._test_template or (self._default_template and
+                                   self._test_template is not False):
+            return TemplatedKeywordCall
+        return KeywordCall
+
+    def _continues(self, value, index):
+        return index > 0 and _Table._continues(self, value, index)
+
+    def _tokenize(self, value, index):
+        if index == 0:
+            if value:
+                self._test_template = None
+            return GherkinTokenizer().tokenize(value, TC_KW_NAME)
+        if index == 1 and self._is_setting(value):
+            if self._is_template(value):
+                self._test_template = False
+                self._tokenizer = self._setting_class(self.set_test_template)
+            else:
+                self._tokenizer = self._setting_class()
+        if index == 1 and self._is_for_loop(value):
+            self._tokenizer = ForLoop()
+        if index == 1 and self._is_empty(value):
+            return [(value, SYNTAX)]
+        return _Table._tokenize(self, value, index)
+
+    def _is_setting(self, value):
+        return value.startswith('[') and value.endswith(']')
+
+    def _is_template(self, value):
+        return normalize(value) == '[template]'
+
+    def _is_for_loop(self, value):
+        return value.startswith(':') and normalize(value, remove=':') == 'for'
+
+    def set_test_template(self, template):
+        self._test_template = self._is_template_set(template)
+
+    def set_default_template(self, template):
+        self._default_template = self._is_template_set(template)
+
+    def _is_template_set(self, template):
+        return normalize(template) not in ('', '\\', 'none', '${empty}')
+
+
+class KeywordTable(TestCaseTable):
+    _tokenizer_class = KeywordCall
+    _setting_class = KeywordSetting
+
+    def _is_template(self, value):
+        return False
+
+
+# Following code copied directly from Robot Framework 2.7.5.
+
+class VariableSplitter:
+
+    def __init__(self, string, identifiers):
+        self.identifier = None
+        self.base = None
+        self.index = None
+        self.start = -1
+        self.end = -1
+        self._identifiers = identifiers
+        self._may_have_internal_variables = False
+        try:
+            self._split(string)
+        except ValueError:
+            pass
+        else:
+            self._finalize()
+
+    def get_replaced_base(self, variables):
+        if self._may_have_internal_variables:
+            return variables.replace_string(self.base)
+        return self.base
+
+    def _finalize(self):
+        self.identifier = self._variable_chars[0]
+        self.base = ''.join(self._variable_chars[2:-1])
+        self.end = self.start + len(self._variable_chars)
+        if self._has_list_variable_index():
+            self.index = ''.join(self._list_variable_index_chars[1:-1])
+            self.end += len(self._list_variable_index_chars)
+
+    def _has_list_variable_index(self):
+        return self._list_variable_index_chars\
+        and self._list_variable_index_chars[-1] == ']'
+
+    def _split(self, string):
+        start_index, max_index = self._find_variable(string)
+        self.start = start_index
+        self._open_curly = 1
+        self._state = self._variable_state
+        self._variable_chars = [string[start_index], '{']
+        self._list_variable_index_chars = []
+        self._string = string
+        start_index += 2
+        for index, char in enumerate(string[start_index:]):
+            index += start_index  # Giving start to enumerate only in Py 2.6+
+            try:
+                self._state(char, index)
+            except StopIteration:
+                return
+            if index  == max_index and not self._scanning_list_variable_index():
+                return
+
+    def _scanning_list_variable_index(self):
+        return self._state in [self._waiting_list_variable_index_state,
+                               self._list_variable_index_state]
+
+    def _find_variable(self, string):
+        max_end_index = string.rfind('}')
+        if max_end_index == -1:
+            raise ValueError('No variable end found')
+        if self._is_escaped(string, max_end_index):
+            return self._find_variable(string[:max_end_index])
+        start_index = self._find_start_index(string, 1, max_end_index)
+        if start_index == -1:
+            raise ValueError('No variable start found')
+        return start_index, max_end_index
+
+    def _find_start_index(self, string, start, end):
+        index = string.find('{', start, end) - 1
+        if index < 0:
+            return -1
+        if self._start_index_is_ok(string, index):
+            return index
+        return self._find_start_index(string, index+2, end)
+
+    def _start_index_is_ok(self, string, index):
+        return string[index] in self._identifiers\
+        and not self._is_escaped(string, index)
+
+    def _is_escaped(self, string, index):
+        escaped = False
+        while index > 0 and string[index-1] == '\\':
+            index -= 1
+            escaped = not escaped
+        return escaped
+
+    def _variable_state(self, char, index):
+        self._variable_chars.append(char)
+        if char == '}' and not self._is_escaped(self._string, index):
+            self._open_curly -= 1
+            if self._open_curly == 0:
+                if not self._is_list_variable():
+                    raise StopIteration
+                self._state = self._waiting_list_variable_index_state
+        elif char in self._identifiers:
+            self._state = self._internal_variable_start_state
+
+    def _is_list_variable(self):
+        return self._variable_chars[0] == '@'
+
+    def _internal_variable_start_state(self, char, index):
+        self._state = self._variable_state
+        if char == '{':
+            self._variable_chars.append(char)
+            self._open_curly += 1
+            self._may_have_internal_variables = True
+        else:
+            self._variable_state(char, index)
+
+    def _waiting_list_variable_index_state(self, char, index):
+        if char != '[':
+            raise StopIteration
+        self._list_variable_index_chars.append(char)
+        self._state = self._list_variable_index_state
+
+    def _list_variable_index_state(self, char, index):
+        self._list_variable_index_chars.append(char)
+        if char == ']':
+            raise StopIteration

vendor/pygments-main/pygments/lexers/_scilab_builtins.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers._scilab_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Builtin list for the ScilabLexer.
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
 # These lists are generated automatically.
 # Run the following in a Scilab script:
 #

vendor/pygments-main/pygments/lexers/_stan_builtins.py

+# -*- coding: utf-8 -*-
+"""
+pygments.lexers._stan_builtins
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This file contains the names of functions for Stan used by
+``pygments.lexers.math.StanLexer.
+
+:copyright: Copyright 2013 by the Pygments team, see AUTHORS.
+:license: BSD, see LICENSE for details.
+"""
+
+KEYWORDS = ['else', 'for', 'if', 'in', 'lower', 'lp__', 'print', 'upper', 'while']
+
+TYPES = [   'corr_matrix',
+    'cov_matrix',
+    'int',
+    'matrix',
+    'ordered',
+    'positive_ordered',
+    'real',
+    'row_vector',
+    'simplex',
+    'unit_vector',
+    'vector']
+
+FUNCTIONS = [   'Phi',
+    'Phi_approx',
+    'abs',
+    'acos',
+    'acosh',
+    'asin',
+    'asinh',
+    'atan',
+    'atan2',
+    'atanh',
+    'bernoulli_cdf',
+    'bernoulli_log',
+    'bernoulli_logit_log',
+    'bernoulli_rng',
+    'beta_binomial_cdf',
+    'beta_binomial_log',
+    'beta_binomial_rng',
+    'beta_cdf',
+    'beta_log',
+    'beta_rng',
+    'binary_log_loss',
+    'binomial_cdf',
+    'binomial_coefficient_log',
+    'binomial_log',
+    'binomial_logit_log',
+    'binomial_rng',
+    'block',
+    'categorical_log',
+    'categorical_rng',
+    'cauchy_cdf',
+    'cauchy_log',
+    'cauchy_rng',
+    'cbrt',
+    'ceil',
+    'chi_square_log',
+    'chi_square_rng',
+    'cholesky_decompose',
+    'col',
+    'cols',
+    'cos',
+    'cosh',
+    'crossprod',
+    'cumulative_sum',
+    'determinant',
+    'diag_matrix',
+    'diag_post_multiply',
+    'diag_pre_multiply',
+    'diagonal',
+    'dims',
+    'dirichlet_log',
+    'dirichlet_rng',
+    'dot_product',
+    'dot_self',
+    'double_exponential_log',
+    'double_exponential_rng',
+    'e',
+    'eigenvalues_sym',
+    'eigenvectors_sym',
+    'epsilon',
+    'erf',
+    'erfc',
+    'exp',
+    'exp2',
+    'exp_mod_normal_cdf',
+    'exp_mod_normal_log',
+    'exp_mod_normal_rng',
+    'expm1',
+    'exponential_cdf',
+    'exponential_log',
+    'exponential_rng',
+    'fabs',
+    'fdim',
+    'floor',
+    'fma',
+    'fmax',
+    'fmin',
+    'fmod',
+    'gamma_log',
+    'gamma_rng',
+    'gumbel_cdf',
+    'gumbel_log',
+    'gumbel_rng',
+    'hypergeometric_log',
+    'hypergeometric_rng',
+    'hypot',
+    'if_else',
+    'int_step',
+    'inv_chi_square_cdf',
+    'inv_chi_square_log',
+    'inv_chi_square_rng',
+    'inv_cloglog',
+    'inv_gamma_cdf',
+    'inv_gamma_log',
+    'inv_gamma_rng',
+    'inv_logit',
+    'inv_wishart_log',
+    'inv_wishart_rng',
+    'inverse',
+    'lbeta',
+    'lgamma',
+    'lkj_corr_cholesky_log',
+    'lkj_corr_cholesky_rng',
+    'lkj_corr_log',
+    'lkj_corr_rng',
+    'lkj_cov_log',
+    'lmgamma',
+    'log',
+    'log10',
+    'log1m',
+    'log1m_inv_logit',
+    'log1p',
+    'log1p_exp',
+    'log2',
+    'log_determinant',
+    'log_inv_logit',
+    'log_sum_exp',
+    'logistic_cdf',
+    'logistic_log',
+    'logistic_rng',
+    'logit',
+    'lognormal_cdf',
+    'lognormal_log',
+    'lognormal_rng',
+    'max',
+    'mdivide_left_tri_low',
+    'mdivide_right_tri_low',
+    'mean',
+    'min',
+    'multi_normal_cholesky_log',
+    'multi_normal_log',
+    'multi_normal_prec_log',
+    'multi_normal_rng',
+    'multi_student_t_log',
+    'multi_student_t_rng',
+    'multinomial_cdf',
+    'multinomial_log',
+    'multinomial_rng',
+    'multiply_log',
+    'multiply_lower_tri_self_transpose',
+    'neg_binomial_cdf',
+    'neg_binomial_log',
+    'neg_binomial_rng',
+    'negative_epsilon',
+    'negative_infinity',
+    'normal_cdf',
+    'normal_log',
+    'normal_rng',
+    'not_a_number',
+    'ordered_logistic_log',
+    'ordered_logistic_rng',
+    'owens_t',
+    'pareto_cdf',
+    'pareto_log',
+    'pareto_rng',
+    'pi',
+    'poisson_cdf',
+    'poisson_log',
+    'poisson_log_log',
+    'poisson_rng',
+    'positive_infinity',
+    'pow',
+    'prod',
+    'rep_array',
+    'rep_matrix',
+    'rep_row_vector',
+    'rep_vector',
+    'round',
+    'row',
+    'rows',
+    'scaled_inv_chi_square_cdf',
+    'scaled_inv_chi_square_log',
+    'scaled_inv_chi_square_rng',
+    'sd',
+    'sin',
+    'singular_values',
+    'sinh',
+    'size',
+    'skew_normal_cdf',
+    'skew_normal_log',
+    'skew_normal_rng',
+    'softmax',
+    'sqrt',
+    'sqrt2',
+    'square',
+    'step',
+    'student_t_cdf',
+    'student_t_log',
+    'student_t_rng',
+    'sum',
+    'tan',
+    'tanh',
+    'tcrossprod',
+    'tgamma',
+    'trace',
+    'trunc',
+    'uniform_log',
+    'uniform_rng',
+    'variance',
+    'weibull_cdf',
+    'weibull_log',
+    'weibull_rng',
+    'wishart_log',
+    'wishart_rng']
+
+DISTRIBUTIONS = [   'bernoulli',
+    'bernoulli_logit',
+    'beta',
+    'beta_binomial',
+    'binomial',
+    'binomial_coefficient',
+    'binomial_logit',
+    'categorical',
+    'cauchy',
+    'chi_square',
+    'dirichlet',
+    'double_exponential',
+    'exp_mod_normal',
+    'exponential',
+    'gamma',
+    'gumbel',
+    'hypergeometric',
+    'inv_chi_square',
+    'inv_gamma',
+    'inv_wishart',
+    'lkj_corr',
+    'lkj_corr_cholesky',
+    'lkj_cov',
+    'logistic',
+    'lognormal',
+    'multi_normal',
+    'multi_normal_cholesky',
+    'multi_normal_prec',
+    'multi_student_t',
+    'multinomial',
+    'multiply',
+    'neg_binomial',
+    'normal',
+    'ordered_logistic',
+    'pareto',
+    'poisson',
+    'poisson_log',
+    'scaled_inv_chi_square',
+    'skew_normal',
+    'student_t',
+    'uniform',
+    'weibull',
+    'wishart']
+
+RESERVED = [   'alignas',
+    'alignof',
+    'and',
+    'and_eq',
+    'asm',
+    'auto',
+    'bitand',
+    'bitor',
+    'bool',
+    'break',
+    'case',
+    'catch',
+    'char',
+    'char16_t',
+    'char32_t',
+    'class',
+    'compl',
+    'const',
+    'const_cast',
+    'constexpr',
+    'continue',
+    'decltype',
+    'default',
+    'delete',
+    'do',
+    'double',
+    'dynamic_cast',
+    'enum',
+    'explicit',
+    'export',
+    'extern',
+    'false',
+    'false',
+    'float',
+    'friend',
+    'goto',
+    'inline',
+    'int',
+    'long',
+    'mutable',
+    'namespace',
+    'new',
+    'noexcept',
+    'not',
+    'not_eq',
+    'nullptr',
+    'operator',
+    'or',
+    'or_eq',
+    'private',
+    'protected',
+    'public',
+    'register',
+    'reinterpret_cast',
+    'repeat',
+    'return',
+    'short',
+    'signed',
+    'sizeof',
+    'static',
+    'static_assert',
+    'static_cast',
+    'struct',
+    'switch',
+    'template',
+    'then',
+    'this',
+    'thread_local',
+    'throw',
+    'true',
+    'true',
+    'try',
+    'typedef',
+    'typeid',
+    'typename',
+    'union',
+    'unsigned',
+    'until',
+    'using',
+    'virtual',
+    'void',
+    'volatile',
+    'wchar_t',
+    'xor',
+    'xor_eq']
+