CHANGELOG

 Changelog
 =========
+0.7 (unreleased)
+------------------
+
+- fix line continuation https://github.com/PyCQA/baron/pull/92 by ibizaman
+- handle corrupt cache file situation https://github.com/PyCQA/baron/pull/76 by ryu2
+- fix special crashing edge case in indentation marker https://github.com/PyCQA/bar by Ahuge
+- fixed incorrect tokenization case "d*e-1". Fixes #85 https://github.com/PyCQA/baron/pull/107 by boxed
+
+Python 3 parsing:
+- support ellipsis https://github.com/PyCQA/baron/pull/121 by odcinek
+- support matrix operator https://github.com/PyCQA/baron/pull/117 by odcinek
+- support f-strings https://github.com/PyCQA/baron/pull/110 by odcinek
+- support numeric literals https://github.com/PyCQA/baron/pull/111 by odcinek
+- support nonlocal statement https://github.com/PyCQA/baron/pull/112 by odcinek
+
 0.6.6 (2017-06-12)
 ------------------
@@ -80,7 +95,7 @@ Changelog
 - bounding box allows you to know the left most and right most position
   of a node see https://baron.readthedocs.io/en/latest/#bounding-box
 - redbaron is classified as supporting python3
-  https://github.com/Psycojoker/baron/pull/51
+  https://github.com/PyCQA/baron/pull/51
 - ensure than when a key is a string, it's empty value is an empty string and
   not None to avoid breaking libs that use introspection to guess the type of
   the key
@@ -137,7 +152,7 @@ Changelog
 - don't add a endl node at the end if not present in the input string
 - de-uniformise call_arguments and function_arguments node, this is just
   creating more problems that anything else
-- fix https://github.com/Psycojoker/redbaron/issues/4
+- fix https://github.com/PyCQA/redbaron/issues/4
 - fix the fact that baron can't parse "{1,}" (but "{1}" is working)
 0.1.1 (2014-03-23)

baron/baron.py

@@ -29,7 +29,7 @@ def _parse(tokens, print_function):
         import traceback
         traceback.print_exc(file=sys.stderr)
         sys.stderr.write("%s\n" % e)
-        sys.stderr.write("\nBaron has failed to parse this input. If this is valid python code (and by that I mean that the python binary successfully parse this code without any syntax error) (also consider that python does not yet parse python 3 code integrally) it would be kind if you can extract a snippet of your code that make Baron fails and open a bug here: https://github.com/Psycojoker/baron/issues\n\nSorry for the inconvenience.")
+        sys.stderr.write("\nBaron has failed to parse this input. If this is valid python code (and by that I mean that the python binary successfully parse this code without any syntax error) (also consider that python does not yet parse python 3 code integrally) it would be kind if you can extract a snippet of your code that make Baron fails and open a bug here: https://github.com/PyCQA/baron/issues\n\nSorry for the inconvenience.")
 def parse(source_code, print_function=None):
@@ -67,4 +67,10 @@ def parse(source_code, print_function=None):
 def tokenize(pouet, print_function=False):
-    return mark_indentation(inner_group(space_group(_tokenize(group(split(pouet)), print_function))))
+    splitted = split(pouet)
+    grouped = group(splitted)
+    print_tokenized = _tokenize(grouped, print_function)
+    space_grouped = space_group(print_tokenized)
+    inner_grouped = inner_group(space_grouped)
+    indentation_marked = mark_indentation(inner_grouped)
+    return indentation_marked

baron/formatting_grouper.py

@@ -24,6 +24,7 @@ BOTH = (
     "DOUBLE_SLASH",
     "PLUS",
     "MINUS",
+    "AT",
     "LEFT_SHIFT",
     "RIGHT_SHIFT",
     "AMPER",
@@ -46,6 +47,7 @@ BOTH = (
     "PLUS_EQUAL",
     "MINUS_EQUAL",
     "STAR_EQUAL",
+    "AT_EQUAL",
     "SLASH_EQUAL",
     "PERCENT_EQUAL",
     "AMPER_EQUAL",
@@ -65,6 +67,8 @@ BOTH = (
 STRING = (
     "STRING",
     "RAW_STRING",
+    "INTERPOLATED_STRING",
+    "INTERPOLATED_RAW_STRING",
     "UNICODE_STRING",
     "UNICODE_RAW_STRING",
     "BINARY_STRING",
@@ -87,6 +91,7 @@ GROUP_SPACE_AFTER = BOTH + (
     "RAISE",
     "EXEC",
     "GLOBAL",
+    "NONLOCAL",
     "PRINT",
     "INDENT",
     "WHILE",

baron/grammator.py

@@ -132,6 +132,7 @@ def generate_parse(print_function):
     @pg.production("small_stmt : assert_stmt")
     @pg.production("small_stmt : raise_stmt")
     @pg.production("small_stmt : global_stmt")
+    @pg.production("small_stmt : nonlocal_stmt")
     @pg.production("compound_stmt : if_stmt")
     @pg.production("compound_stmt : while_stmt")
     @pg.production("compound_stmt : for_stmt")
@@ -655,10 +656,12 @@ def generate_parse(print_function):
     # TODO tests those other kind of strings
     @pg.production("string : STRING")
     @pg.production("string : RAW_STRING")
+    @pg.production("string : INTERPOLATED_STRING")
     @pg.production("string : UNICODE_STRING")
     @pg.production("string : BINARY_STRING")
     @pg.production("string : UNICODE_RAW_STRING")
     @pg.production("string : BINARY_RAW_STRING")
+    @pg.production("string : INTERPOLATED_RAW_STRING")
     def string(pack):
         (string_,) = pack
         return [{

baron/grammator_operators.py

@@ -34,6 +34,7 @@ def include_operators(pg):
     @pg.production("augassign_operator : SLASH_EQUAL")
     @pg.production("augassign_operator : PERCENT_EQUAL")
     @pg.production("augassign_operator : AMPER_EQUAL")
+    @pg.production("augassign_operator : AT_EQUAL")
     @pg.production("augassign_operator : VBAR_EQUAL")
     @pg.production("augassign_operator : CIRCUMFLEX_EQUAL")
     @pg.production("augassign_operator : LEFT_SHIFT_EQUAL")
@@ -147,6 +148,7 @@ def include_operators(pg):
     @pg.production("term : factor SLASH term")
     @pg.production("term : factor PERCENT term")
     @pg.production("term : factor DOUBLE_SLASH term")
+    @pg.production("term : factor AT term")
     @pg.production("power : atom DOUBLE_STAR factor")
     @pg.production("power : atom DOUBLE_STAR power")
     def binary_operator_node(pack):
@@ -254,6 +256,7 @@ def include_operators(pg):
         }]
     @pg.production("subscript : DOT DOT DOT")
+    @pg.production("atom : DOT DOT DOT")
     def subscript_ellipsis(pack):
         (dot1, dot2, dot3) = pack
         return {

baron/grammator_primitives.py

@@ -235,6 +235,16 @@ def include_primivites(pg, print_function):
         }
+    @pg.production("nonlocal_stmt : NONLOCAL names")
+    def nonlocal_stmt(pack):
+        (token, names) = pack
+        return {
+            "type": "nonlocal",
+            "formatting": token.hidden_tokens_after,
+            "value": names,
+        }
+
+
     @pg.production("names : NAME")
     def names_name(pack):
         (name,) = pack

baron/grouper.py

@@ -12,6 +12,7 @@ to_group = (
     ("&", "="),
     ("|", "="),
     ("^", "="),
+    ("@", "="),
     ("/", "/"),
     ("*", "*"),
     ("<", "<"),
@@ -51,9 +52,9 @@ def group_generator(sequence):
             current += next(iterator)
         if current in to_group_keys and matching_found(to_group, current, iterator.show_next()):
             current += next(iterator)
-        if current in list('uUrRbB') and str(iterator.show_next()).startswith(('"', "'")):
+        if current in list('uUfFrRbB') and str(iterator.show_next()).startswith(('"', "'")):
             current += next(iterator)
-        if str(current).lower() in ["ur", "br"] and str(iterator.show_next()).startswith(('"', "'")):
+        if str(current).lower() in ["ur", "br", "fr", "rf"] and str(iterator.show_next()).startswith(('"', "'")):
             current += next(iterator)
         if any([re.match(x, current) for x in (r'^\d+[eE]$', r'^\d+\.\d*[eE]$', r'^\.\d+[eE]$')]):
             current += next(iterator)
@@ -95,7 +96,7 @@ def group_generator(sequence):
         if re.match(r'^\d+\.?[eE]$', current) and match_on_next(r'^\d+$', iterator):
             current += next(iterator)
-        if re.match(r'^\d*\.?\d*[eE]$', current) and match_on_next(r'^[-+]$', iterator) and iterator.show_next(2) and re.match(r'^\d+$', iterator.show_next(2)):
+        if re.match(r'^\d*\.?\d*[eE]$', current) and not re.match('[eE]', current) and match_on_next(r'^[-+]$', iterator) and iterator.show_next(2) and re.match(r'^\d+$', iterator.show_next(2)):
             current += next(iterator)
             current += next(iterator)

baron/indentation_marker.py

@@ -35,7 +35,7 @@ def get_space(node):
     a ('ENDL', '\n') node - then we return None as a flag value. This is
     maybe not the best behavior but it seems to work for now.
     """
-    if len(node) < 3 or len(node[3]) == 0:
+    if len(node) < 4 or len(node[3]) == 0:
         return None
     return transform_tabs_to_spaces(node[3][0][1])

baron/inner_formatting_grouper.py

@@ -35,6 +35,8 @@ GROUP_ON = (
     # TODO test everything bellow
     "STRING",
     "RAW_STRING",
+    "INTERPOLATED_STRING",
+    "INTERPOLATED_RAW_STRING",
     "BINARY_STRING",
     "BINARY_RAW_STRING",
     "UNICODE_STRING",
@@ -67,11 +69,13 @@ GROUP_ON = (
     "NOT",
     "AND",
     "OR",
+    "AT",
     "IF",
     "ELSE",
     "EQUAL",
     "PLUS_EQUAL",
     "MINUS_EQUAL",
+    "AT_EQUAL",
     "STAR_EQUAL",
     "SLASH_EQUAL",
     "PERCENT_EQUAL",
@@ -178,15 +182,4 @@ def group_generator(sequence):
                     debug_file_content += _append_to_debug_file_content(iterator.show_next())
                     current = append_to_token_after(current, [next(iterator)])
-
-        if current[0] == "SPACE":
-            debug_file_content = debug_file_content.split("\n")
-            debug_file_content = list(zip(range(1, len(debug_file_content) + 1), debug_file_content))
-            debug_file_content = debug_file_content[-3:]
-            debug_file_content = "\n".join(["%4s %s" % (x[0], x[1]) for x in debug_file_content])
-            debug_file_content += "<--- here"
-            debug_text = "Unexpected '%s' token:\n\n" % current[0].lower() + debug_file_content + "\n\n"
-            debug_text += "Should have been grouped on either %s (before) or %s (after) token." % (debug_previous_token, iterator.show_next())
-            raise UnExpectedFormattingToken(debug_text)
-
         yield current

baron/parser.py

@@ -63,16 +63,23 @@ class BaronParserGenerator(ParserGenerator):
         table = None
         if os.path.exists(cache_file):
             with open(cache_file) as f:
-                data = json.load(f)
-                stat_result = os.fstat(f.fileno())
-            if (
-                os.name == "nt" or (
-                    stat_result.st_uid == os.getuid() and
-                    stat.S_IMODE(stat_result.st_mode) == 0o0600
-                )
-            ):
-                if self.data_is_valid(g, data):
-                    table = LRTable.from_cache(g, data)
+                try:
+                    data = json.load(f)
+                except:
+                    os.remove(cache_file)
+                    data = None
+
+                if data is not None:
+                    stat_result = os.fstat(f.fileno())
+                    if (
+                        os.name == "nt" or (
+                            stat_result.st_uid == os.getuid() and
+                            stat.S_IMODE(stat_result.st_mode) == 0o0600
+                        )
+                    ):
+                        if self.data_is_valid(g, data):
+                            table = LRTable.from_cache(g, data)
+
         if table is None:
             table = LRTable.from_grammar(g)
             try:
@@ -164,5 +171,5 @@ class BaronLRParser(LRParser):
                     debug_output += "<---- here"
                     debug_output = "Error, got an unexpected token %s here:\n\n" % ltype + debug_output
                     debug_output += "\n\nThe token %s should be one of those: %s" % (ltype, ", ".join(sorted(self.lr_table.lr_action[current_state].keys())))
-                    debug_output += "\n\nIt is not normal that you see this error, it means that Baron has failed to parse valid Python code. It would be kind if you can extract the snippet of your code that makes Baron fail and open a bug here: https://github.com/Psycojoker/baron/issues\n\nSorry for the inconvenience."
+                    debug_output += "\n\nIt is not normal that you see this error, it means that Baron has failed to parse valid Python code. It would be kind if you can extract the snippet of your code that makes Baron fail and open a bug here: https://github.com/PyCQA/baron/issues\n\nSorry for the inconvenience."
                     raise ParsingError(debug_output)

baron/render.py

@@ -504,7 +504,11 @@ nodes_rendering_order = {
             ("formatting", "second_formatting", "as"),
             ("key",        "as",                "as"),
         ],
-
+        "nonlocal": [
+            ("constant",    "nonlocal",         True),
+            ("formatting", "formatting",        True),
+            ("list",        "value",            True),
+        ],
         "del": [
             ("constant",   "del",               True),
             ("formatting", "formatting",        True),

baron/tokenizer.py

@@ -5,7 +5,7 @@ from .utils import BaronError
 class UnknowItem(BaronError):
     pass
-KEYWORDS = ("and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield")
+KEYWORDS = ("and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "nonlocal", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield")
 TOKENS = (
     (r'[a-zA-Z_]\w*', 'NAME'),
@@ -20,14 +20,14 @@ TOKENS = (
     (r'\d+\.[jJ]', 'COMPLEX'),
     (r'\d+[jJ]', 'COMPLEX'),
     (r'\d+\.', 'FLOAT'),
-    (r'\d*\.\d+[lL]?', 'FLOAT'),
-    (r'\d+\.\d*[lL]?', 'FLOAT'),
+    (r'\d*[_\d]*\.[_\d]+[lL]?', 'FLOAT'),
+    (r'\d+[_\d]+\.[_\d]*[lL]?', 'FLOAT'),
     (r'\.', 'DOT'),
-    (r'[1-9]+\d*[lL]', 'LONG'),
-    (r'[1-9]+\d*', 'INT'),
-    (r'0[xX][\da-fA-F]+[lL]?', 'HEXA'),
-    (r'(0[oO][0-7]+)|(0[0-7]*)[lL]?', 'OCTA'),
-    (r'0[bB][01]+[lL]?', 'BINARY'),
+    (r'[1-9]+[_\d]*[lL]', 'LONG'),
+    (r'[1-9]+[_\d]*', 'INT'),
+    (r'0[xX][\d_a-fA-F]+[lL]?', 'HEXA'),
+    (r'(0[oO][0-7]+)|(0[0-7_]*)[lL]?', 'OCTA'),
+    (r'0[bB][01_]+[lL]?', 'BINARY'),
     (r'\(', 'LEFT_PARENTHESIS'),
     (r'\)', 'RIGHT_PARENTHESIS'),
     (r':', 'COLON'),
@@ -40,6 +40,7 @@ TOKENS = (
     (r'/', 'SLASH'),
     (r'\|', 'VBAR'),
     (r'&', 'AMPER'),
+    (r'@', 'AT'),
     (r'<', 'LESS'),
     (r'>', 'GREATER'),
     (r'=', 'EQUAL'),
@@ -61,6 +62,7 @@ TOKENS = (
     (r'\*\*', 'DOUBLE_STAR'),
     (r'\+=', 'PLUS_EQUAL'),
     (r'-=', 'MINUS_EQUAL'),
+    (r'@=', 'AT_EQUAL'),
     (r'\*=', 'STAR_EQUAL'),
     (r'/=', 'SLASH_EQUAL'),
     (r'%=', 'PERCENT_EQUAL'),
@@ -78,10 +80,13 @@ TOKENS = (
     (r'(\s|\\\n|\\\r\n)+', 'SPACE'),
     (r'["\'](.|\n|\r)*["\']', 'STRING'),
     (r'[uU]["\'](.|\n|\r)*["\']', 'UNICODE_STRING'),
+    (r'[fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_STRING'),
     (r'[rR]["\'](.|\n|\r)*["\']', 'RAW_STRING'),
     (r'[bB]["\'](.|\n|\r)*["\']', 'BINARY_STRING'),
     (r'[uU][rR]["\'](.|\n|\r)*["\']', 'UNICODE_RAW_STRING'),
     (r'[bB][rR]["\'](.|\n|\r)*["\']', 'BINARY_RAW_STRING'),
+    (r'[fF][rR]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
+    (r'[rR][fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
 )

docs/grammar.rst

+Grammar reference
+=================
+
+This page is here to serve as a reference for the grammar implementation. Baron
+started as a python2.7 grammar implementation following `the official
+specification for that <https://docs.python.org/2/reference/grammar.html>`_ and
+supporting both :file:`print statement` and :file:`print function`.
+
+The evolution path regarding python3* is the adopt the same strategy that
+lib2to3 and try to support a combination of both grammar as much as possible.
+
+This page describe the decisions taken regarding this dual support and it's
+progress. Hopefully there will be very few conflicting situations.
+
+Current goal is `python 3.6 specification <https://docs.python.org/3.6/reference/grammar.html>`_.
+
+Python 2 and python 3.6 grammar differences
+===========================================
+
+As a reference and an overview, here is screenshot of vimdiff showing the difference between python 2.7 and python 3.6 grammar differences.
+
+.. image:: grammar-python-2.7-3.6-diff-1.png
+
+.. image:: grammar-python-2.7-3.6-diff-2.png
+
+.. image:: grammar-python-2.7-3.6-diff-3.png
+
+List of differences
+===================
+
+**Some of the diff have been edited to isolate the focused difference of the
+section**
+
+Python 3.3 is the based grammar I've started diffing with, some of the grammar
+differences marked as 3.3 are actually from older python version.
+
+Current status
+==============
+
+Grammar diff has been done up to python 3.6.2.
+
+Still, some stuff for the lexer are probably missing in this list like:
+
+* fstrings
+* adding _ in numbers
+
+I need to got through all release notes to see that.
+
+Done
+====
+
+Print function
+--------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/print_function.png
+
+Already done since the start.
+
+This is handle at the parser initialisation level, is activate or not the
+print_function rule.
+
+Ellipsis as atom (to replace 'pass')
+------------------------------------
+
+Python 3.3 or earlier
+RedBaron: probably nothing to do (to be confirmed)
+
+Baron: https://github.com/PyCQA/baron/pull/121
+
+::
+
+    ADD '...' in atom rule
+
+Matrix operator
+---------------
+
+Python 3.5
+
+Baron: https://github.com/PyCQA/baron/pull/117
+RedBaron: probably nothing to do (to be confirmed)
+
+::
+
+    ADD '@' and '@=' to the lexer
+    ADD '@=' in augassign
+    ADD '@' in term
+
+::
+
+    augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+                '<<=' | '>>=' | '**=' | '//=')
+
+    term: factor (('*'|'@'|'/'|'%'|'//') factor)*
+
+
+f-strings
+---------
+
+Python 3.6
+
+Baron: https://github.com/PyCQA/baron/pull/110
+RedBaron: TODO
+
+::
+
+    add f'' rf'' fr'' to the lexer
+
+Underscore in numeric literals
+------------------------------
+
+Python 3.6
+
+Baron: https://github.com/PyCQA/baron/pull/111
+RedBaron: probably nothing to do (to be confirmed)
+
+::
+
+    modify lexer to handle this
+
+Nonlocal statement
+------------------
+
+Python 3.3 or earlier
+
+Baron: https://github.com/PyCQA/baron/pull/112
+RedBaron: TODO
+
+.. image:: ./grammar_diff/nonlocal_statement.png
+
+Action:
+
+::
+
+    ADD 'nonlocal' to lexer
+    ADD 'nonlocal_stmt' to 'small_stmt'
+
+    ADD new rule:
+    nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+
+TODO
+====
+
+Ellipsis in from import
+-----------------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/ellipsis_in_from_import.png
+
+Typed arguments
+---------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/typed_args.png
+
+Action:
+
+::
+
+    # parameters
+    # this is mixed with the removal of def a((b, c)): style
+    # which will probably need to continue supporting
+
+    CHANGE parameters: '(' [varargslist] ')'
+                               ^
+    TO parameters: '(' [typedargslist] ')'
+                             ^
+
+::
+
+    # CHANGE
+    varargslist: ((fpdef ['=' test] ',')*
+                  ('*' NAME [',' '**' NAME] | '**' NAME) |
+                  fpdef ['=' test] (',' fpdef ['=' test])* [','])
+    fpdef: NAME | '(' fplist ')'
+    fplist: fpdef (',' fpdef)* [',']
+
+    # TO
+    typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
+            '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+          | '**' tfpdef [',']]]
+      | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [','])
+    tfpdef: NAME [':' test]
+    varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+            '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+          | '**' vfpdef [',']]]
+      | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']
+    )
+    vfpdef: NAME
+
+
+
+Function return type
+--------------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/function_return_type.png
+
+Action:
+
+::
+
+    ADD '->' to the lexer
+    ADD ['->' test] to funcdef rule
+    funcdef: 'def' NAME parameters ['->' test] ':' suite
+
+Exec function
+-------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/exec_function.png
+
+Like print_function but for 'exec'.
+
+No one seems to be using that.
+
+*var generalisation
+-------------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/testlist_start_expressiong.png
+
+.
+
+.. image:: ./grammar_diff/star_expr.png
+
+.
+
+.. image:: ./grammar_diff/star_expr_in_testlist_comp.png
+
+.
+
+.. image:: ./grammar_diff/star_expr_in_expr_list.png
+
+Raise from
+----------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/raise_from.png
+
+Action:
+
+::
+
+    # 2.7
+    raise_stmt: 'raise' [test [',' test [',' test]]]
+
+    # 3.3
+    raise_stmt: 'raise' [test ['from' test]]
+
+    # merge
+    raise_stmt: 'raise' [test [(',' test [',' test]] | 'from' test)]
+
+New lambda grammar
+------------------
+
+Python 3.3 or earlier
+
+I have no idea on what to do with this one yet.
+
+.. image:: ./grammar_diff/new_lambda_grammar.png
+
+.. image:: ./grammar_diff/new_grammar_for_if_cond.png
+
+Remove old list comprehension syntax
+------------------------------------
+
+Python 3.3 or earlier
+
+I'm not sure on how to handle both situations (and it is needed? Old list
+comprehension syntax is like super edgy, I really wonder if anyonne has
+actually used that one that?)
+
+.. image:: ./grammar_diff/remove_old_list_comprehension_syntax.png
+
+.. image:: ./grammar_diff/no_more_list_for_rule.png
+
+False|True|None|... are now atoms in the grammar
+------------------------------------------------
+
+Python 3.3 or earlier
+
+Do I need to do anything about that?
+
+.. image:: ./grammar_diff/more_atoms.png
+
+Inheritance in class definition uses arglist now
+------------------------------------------------
+
+Python 3.3 or earlier
+
+I have no idea on why this is here but that's easy to change.
+
+.. image:: ./grammar_diff/class_inherit_is_arglist_now.png
+
+Yield From
+----------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/yield_from.png
+
+Async Funcdef
+-------------
+
+Python 3.5
+
+Before:
+
+::
+
+    decorated: decorators (classdef | funcdef)
+
+After:
+
+::
+
+    decorated: decorators (classdef | funcdef | async_funcdef)
+    async_funcdef: ASYNC funcdef
+
+
+Await atom
+----------
+
+Python 3.5
+
+Before:
+
+::
+
+    power: atom trailer* ['**' factor]
+
+After:
+
+::
+
+    power: atom_expr ['**' factor]
+    atom_expr: [AWAIT] atom trailer*
+
+Kwargs expressions
+------------------
+
+Python 3.5
+
+Before:
+
+::
+
+    dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
+                      (test (comp_for | (',' test)* [','])) )
+
+    arglist: (argument ',')* (argument [',']
+                             |'*' test (',' argument)* [',' '**' test]
+                             |'**' test)
+
+    # The reason that keywords are test nodes instead of NAME is that using NAME
+    # results in an ambiguity. ast.c makes sure it's a NAME.
+    argument: test [comp_for] | test '=' test
+
+After:
+
+::
+
+    dictorsetmaker: ( ((test ':' test | '**' expr)
+                       (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                      ((test | star_expr)
+                       (comp_for | (',' (test | star_expr))* [','])) )
+
+    # can be simplified apparently
+    arglist: argument (',' argument)*  [',']
+
+    # The reason that keywords are test nodes instead of NAME is that using NAME
+    # results in an ambiguity. ast.c makes sure it's a NAME.
+    # "test '=' test" is really "keyword '=' test", but we have no such token.
+    # These need to be in a single rule to avoid grammar that is ambiguous
+    # to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+    # we explicitly match '*' here, too, to give it proper precedence.
+    # Illegal combinations and orderings are blocked in ast.c:
+    # multiple (test comp_for) arguments are blocked; keyword unpackings
+    # that precede iterable unpackings are blocked; etc.
+    argument: ( test [comp_for] |
+                test '=' test |
+                '**' test |
+                '*' test )
+
+
+
+Variables annotations
+---------------------
+
+Python 3.6
+
+Before:
+
+::
+
+    expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+                         ('=' (yield_expr|testlist_star_expr))*)
+
+After:
+
+::
+
+    expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
+                         ('=' (yield_expr|testlist_star_expr))*)
+    annassign: ':' test ['=' test]
+
+async for loop
+--------------
+
+Python 3.6
+
+Before:
+
+::
+
+    comp_for: 'for' exprlist 'in' or_test [comp_iter]
+
+After:
+
+::
+
+    comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
+
+
+Refactoring in typedargslist ?
+------------------------------
+
+I think this is for asynchronous generator and comprehension:
+
+* https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep525
+* https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep530
+
+Before:
+
+::
+
+    typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
+           ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
+         |  '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+    varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
+           ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
+         |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
+
+After:
+
+::
+
+    typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
+           ['*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+          | '**' tfpdef [',']]]
+      | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [','])
+    varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
+           ['*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+         | '**' vfpdef [',']]]
+      | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']
+    )
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Nothing to do
+=============
+
+Those are things that have been removed from python3 grammar but we still need
+to support (and we already do) so we don't have to do anything.
+
+No more commat syntax in except close
+-------------------------------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/no_more_commat_in_execption_close.png
+
+No more backquote syntax
+------------------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/no_more_backquote_syntax.png
+
+No more '.' '.' '.' in the grammar
+----------------------------------
+
+Python 3.3 or earlier
+
+.. image:: ./grammar_diff/ellipsis_is_first_class_now_not_needed_anymore.png