Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • pycqa/baron
1 result
Show changes
Commits on Source (42)
Hide whitespace changes
Inline Side-by-side
Showing
with 761 additions and 87 deletions
CHANGELOG
View file @ e781669e
Changelog
=========
 
0.7 (unreleased)
------------------
- fix line continuation https://github.com/PyCQA/baron/pull/92 by ibizaman
- handle corrupt cache file situation https://github.com/PyCQA/baron/pull/76 by ryu2
- fix special crashing edge case in indentation marker https://github.com/PyCQA/bar by Ahuge
- fixed incorrect tokenization case "d*e-1". Fixes #85 https://github.com/PyCQA/baron/pull/107 by boxed
Python 3 parsing:
- support ellipsis https://github.com/PyCQA/baron/pull/121 by odcinek
- support matrix operator https://github.com/PyCQA/baron/pull/117 by odcinek
- support f-strings https://github.com/PyCQA/baron/pull/110 by odcinek
- support numeric literals https://github.com/PyCQA/baron/pull/111 by odcinek
- support nonlocal statement https://github.com/PyCQA/baron/pull/112 by odcinek
0.6.6 (2017-06-12)
------------------
 
@@ -80,7 +95,7 @@ Changelog
- bounding box allows you to know the left most and right most position
of a node see https://baron.readthedocs.io/en/latest/#bounding-box
- redbaron is classified as supporting python3
https://github.com/Psycojoker/baron/pull/51
https://github.com/PyCQA/baron/pull/51
- ensure than when a key is a string, it's empty value is an empty string and
not None to avoid breaking libs that use introspection to guess the type of
the key
@@ -137,7 +152,7 @@ Changelog
- don't add a endl node at the end if not present in the input string
- de-uniformise call_arguments and function_arguments node, this is just
creating more problems that anything else
- fix https://github.com/Psycojoker/redbaron/issues/4
- fix https://github.com/PyCQA/redbaron/issues/4
- fix the fact that baron can't parse "{1,}" (but "{1}" is working)
 
0.1.1 (2014-03-23)
baron/baron.py
View file @ e781669e
@@ -29,7 +29,7 @@ def _parse(tokens, print_function):
import traceback
traceback.print_exc(file=sys.stderr)
sys.stderr.write("%s\n" % e)
sys.stderr.write("\nBaron has failed to parse this input. If this is valid python code (and by that I mean that the python binary successfully parse this code without any syntax error) (also consider that python does not yet parse python 3 code integrally) it would be kind if you can extract a snippet of your code that make Baron fails and open a bug here: https://github.com/Psycojoker/baron/issues\n\nSorry for the inconvenience.")
sys.stderr.write("\nBaron has failed to parse this input. If this is valid python code (and by that I mean that the python binary successfully parse this code without any syntax error) (also consider that python does not yet parse python 3 code integrally) it would be kind if you can extract a snippet of your code that make Baron fails and open a bug here: https://github.com/PyCQA/baron/issues\n\nSorry for the inconvenience.")
 
 
def parse(source_code, print_function=None):
@@ -67,4 +67,10 @@ def parse(source_code, print_function=None):
 
 
def tokenize(pouet, print_function=False):
return mark_indentation(inner_group(space_group(_tokenize(group(split(pouet)), print_function))))
splitted = split(pouet)
grouped = group(splitted)
print_tokenized = _tokenize(grouped, print_function)
space_grouped = space_group(print_tokenized)
inner_grouped = inner_group(space_grouped)
indentation_marked = mark_indentation(inner_grouped)
return indentation_marked
baron/formatting_grouper.py
View file @ e781669e
@@ -24,6 +24,7 @@ BOTH = (
"DOUBLE_SLASH",
"PLUS",
"MINUS",
"AT",
"LEFT_SHIFT",
"RIGHT_SHIFT",
"AMPER",
@@ -46,6 +47,7 @@ BOTH = (
"PLUS_EQUAL",
"MINUS_EQUAL",
"STAR_EQUAL",
"AT_EQUAL",
"SLASH_EQUAL",
"PERCENT_EQUAL",
"AMPER_EQUAL",
@@ -65,6 +67,8 @@ BOTH = (
STRING = (
"STRING",
"RAW_STRING",
"INTERPOLATED_STRING",
"INTERPOLATED_RAW_STRING",
"UNICODE_STRING",
"UNICODE_RAW_STRING",
"BINARY_STRING",
@@ -87,6 +91,7 @@ GROUP_SPACE_AFTER = BOTH + (
"RAISE",
"EXEC",
"GLOBAL",
"NONLOCAL",
"PRINT",
"INDENT",
"WHILE",
baron/grammator.py
View file @ e781669e
@@ -132,6 +132,7 @@ def generate_parse(print_function):
@pg.production("small_stmt : assert_stmt")
@pg.production("small_stmt : raise_stmt")
@pg.production("small_stmt : global_stmt")
@pg.production("small_stmt : nonlocal_stmt")
@pg.production("compound_stmt : if_stmt")
@pg.production("compound_stmt : while_stmt")
@pg.production("compound_stmt : for_stmt")
@@ -655,10 +656,12 @@ def generate_parse(print_function):
# TODO tests those other kind of strings
@pg.production("string : STRING")
@pg.production("string : RAW_STRING")
@pg.production("string : INTERPOLATED_STRING")
@pg.production("string : UNICODE_STRING")
@pg.production("string : BINARY_STRING")
@pg.production("string : UNICODE_RAW_STRING")
@pg.production("string : BINARY_RAW_STRING")
@pg.production("string : INTERPOLATED_RAW_STRING")
def string(pack):
(string_,) = pack
return [{
baron/grammator_operators.py
View file @ e781669e
@@ -34,6 +34,7 @@ def include_operators(pg):
@pg.production("augassign_operator : SLASH_EQUAL")
@pg.production("augassign_operator : PERCENT_EQUAL")
@pg.production("augassign_operator : AMPER_EQUAL")
@pg.production("augassign_operator : AT_EQUAL")
@pg.production("augassign_operator : VBAR_EQUAL")
@pg.production("augassign_operator : CIRCUMFLEX_EQUAL")
@pg.production("augassign_operator : LEFT_SHIFT_EQUAL")
@@ -147,6 +148,7 @@ def include_operators(pg):
@pg.production("term : factor SLASH term")
@pg.production("term : factor PERCENT term")
@pg.production("term : factor DOUBLE_SLASH term")
@pg.production("term : factor AT term")
@pg.production("power : atom DOUBLE_STAR factor")
@pg.production("power : atom DOUBLE_STAR power")
def binary_operator_node(pack):
@@ -254,6 +256,7 @@ def include_operators(pg):
}]
 
@pg.production("subscript : DOT DOT DOT")
@pg.production("atom : DOT DOT DOT")
def subscript_ellipsis(pack):
(dot1, dot2, dot3) = pack
return {
baron/grammator_primitives.py
View file @ e781669e
@@ -235,6 +235,16 @@ def include_primivites(pg, print_function):
}
 
 
@pg.production("nonlocal_stmt : NONLOCAL names")
def nonlocal_stmt(pack):
(token, names) = pack
return {
"type": "nonlocal",
"formatting": token.hidden_tokens_after,
"value": names,
}
@pg.production("names : NAME")
def names_name(pack):
(name,) = pack
baron/grouper.py
View file @ e781669e
@@ -12,6 +12,7 @@ to_group = (
("&", "="),
("|", "="),
("^", "="),
("@", "="),
("/", "/"),
("*", "*"),
("<", "<"),
@@ -51,9 +52,9 @@ def group_generator(sequence):
current += next(iterator)
if current in to_group_keys and matching_found(to_group, current, iterator.show_next()):
current += next(iterator)
if current in list('uUrRbB') and str(iterator.show_next()).startswith(('"', "'")):
if current in list('uUfFrRbB') and str(iterator.show_next()).startswith(('"', "'")):
current += next(iterator)
if str(current).lower() in ["ur", "br"] and str(iterator.show_next()).startswith(('"', "'")):
if str(current).lower() in ["ur", "br", "fr", "rf"] and str(iterator.show_next()).startswith(('"', "'")):
current += next(iterator)
if any([re.match(x, current) for x in (r'^\d+[eE]$', r'^\d+\.\d*[eE]$', r'^\.\d+[eE]$')]):
current += next(iterator)
@@ -95,7 +96,7 @@ def group_generator(sequence):
if re.match(r'^\d+\.?[eE]$', current) and match_on_next(r'^\d+$', iterator):
current += next(iterator)
 
if re.match(r'^\d*\.?\d*[eE]$', current) and match_on_next(r'^[-+]$', iterator) and iterator.show_next(2) and re.match(r'^\d+$', iterator.show_next(2)):
if re.match(r'^\d*\.?\d*[eE]$', current) and not re.match('[eE]', current) and match_on_next(r'^[-+]$', iterator) and iterator.show_next(2) and re.match(r'^\d+$', iterator.show_next(2)):
current += next(iterator)
current += next(iterator)
 
baron/indentation_marker.py
View file @ e781669e
@@ -35,7 +35,7 @@ def get_space(node):
a ('ENDL', '\n') node - then we return None as a flag value. This is
maybe not the best behavior but it seems to work for now.
"""
if len(node) < 3 or len(node[3]) == 0:
if len(node) < 4 or len(node[3]) == 0:
return None
return transform_tabs_to_spaces(node[3][0][1])
 
baron/inner_formatting_grouper.py
View file @ e781669e
@@ -35,6 +35,8 @@ GROUP_ON = (
# TODO test everything bellow
"STRING",
"RAW_STRING",
"INTERPOLATED_STRING",
"INTERPOLATED_RAW_STRING",
"BINARY_STRING",
"BINARY_RAW_STRING",
"UNICODE_STRING",
@@ -67,11 +69,13 @@ GROUP_ON = (
"NOT",
"AND",
"OR",
"AT",
"IF",
"ELSE",
"EQUAL",
"PLUS_EQUAL",
"MINUS_EQUAL",
"AT_EQUAL",
"STAR_EQUAL",
"SLASH_EQUAL",
"PERCENT_EQUAL",
@@ -178,15 +182,4 @@ def group_generator(sequence):
debug_file_content += _append_to_debug_file_content(iterator.show_next())
current = append_to_token_after(current, [next(iterator)])
 
if current[0] == "SPACE":
debug_file_content = debug_file_content.split("\n")
debug_file_content = list(zip(range(1, len(debug_file_content) + 1), debug_file_content))
debug_file_content = debug_file_content[-3:]
debug_file_content = "\n".join(["%4s %s" % (x[0], x[1]) for x in debug_file_content])
debug_file_content += "<--- here"
debug_text = "Unexpected '%s' token:\n\n" % current[0].lower() + debug_file_content + "\n\n"
debug_text += "Should have been grouped on either %s (before) or %s (after) token." % (debug_previous_token, iterator.show_next())
raise UnExpectedFormattingToken(debug_text)
yield current
baron/parser.py
View file @ e781669e
@@ -63,16 +63,23 @@ class BaronParserGenerator(ParserGenerator):
table = None
if os.path.exists(cache_file):
with open(cache_file) as f:
data = json.load(f)
stat_result = os.fstat(f.fileno())
if (
os.name == "nt" or (
stat_result.st_uid == os.getuid() and
stat.S_IMODE(stat_result.st_mode) == 0o0600
)
):
if self.data_is_valid(g, data):
table = LRTable.from_cache(g, data)
try:
data = json.load(f)
except:
os.remove(cache_file)
data = None
if data is not None:
stat_result = os.fstat(f.fileno())
if (
os.name == "nt" or (
stat_result.st_uid == os.getuid() and
stat.S_IMODE(stat_result.st_mode) == 0o0600
)
):
if self.data_is_valid(g, data):
table = LRTable.from_cache(g, data)
if table is None:
table = LRTable.from_grammar(g)
try:
@@ -164,5 +171,5 @@ class BaronLRParser(LRParser):
debug_output += "<---- here"
debug_output = "Error, got an unexpected token %s here:\n\n" % ltype + debug_output
debug_output += "\n\nThe token %s should be one of those: %s" % (ltype, ", ".join(sorted(self.lr_table.lr_action[current_state].keys())))
debug_output += "\n\nIt is not normal that you see this error, it means that Baron has failed to parse valid Python code. It would be kind if you can extract the snippet of your code that makes Baron fail and open a bug here: https://github.com/Psycojoker/baron/issues\n\nSorry for the inconvenience."
debug_output += "\n\nIt is not normal that you see this error, it means that Baron has failed to parse valid Python code. It would be kind if you can extract the snippet of your code that makes Baron fail and open a bug here: https://github.com/PyCQA/baron/issues\n\nSorry for the inconvenience."
raise ParsingError(debug_output)
baron/render.py
View file @ e781669e
@@ -504,7 +504,11 @@ nodes_rendering_order = {
("formatting", "second_formatting", "as"),
("key", "as", "as"),
],
"nonlocal": [
("constant", "nonlocal", True),
("formatting", "formatting", True),
("list", "value", True),
],
"del": [
("constant", "del", True),
("formatting", "formatting", True),
baron/tokenizer.py
View file @ e781669e
@@ -5,7 +5,7 @@ from .utils import BaronError
class UnknowItem(BaronError):
pass
 
KEYWORDS = ("and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield")
KEYWORDS = ("and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "nonlocal", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield")
 
TOKENS = (
(r'[a-zA-Z_]\w*', 'NAME'),
@@ -20,14 +20,14 @@ TOKENS = (
(r'\d+\.[jJ]', 'COMPLEX'),
(r'\d+[jJ]', 'COMPLEX'),
(r'\d+\.', 'FLOAT'),
(r'\d*\.\d+[lL]?', 'FLOAT'),
(r'\d+\.\d*[lL]?', 'FLOAT'),
(r'\d*[_\d]*\.[_\d]+[lL]?', 'FLOAT'),
(r'\d+[_\d]+\.[_\d]*[lL]?', 'FLOAT'),
(r'\.', 'DOT'),
(r'[1-9]+\d*[lL]', 'LONG'),
(r'[1-9]+\d*', 'INT'),
(r'0[xX][\da-fA-F]+[lL]?', 'HEXA'),
(r'(0[oO][0-7]+)|(0[0-7]*)[lL]?', 'OCTA'),
(r'0[bB][01]+[lL]?', 'BINARY'),
(r'[1-9]+[_\d]*[lL]', 'LONG'),
(r'[1-9]+[_\d]*', 'INT'),
(r'0[xX][\d_a-fA-F]+[lL]?', 'HEXA'),
(r'(0[oO][0-7]+)|(0[0-7_]*)[lL]?', 'OCTA'),
(r'0[bB][01_]+[lL]?', 'BINARY'),
(r'\(', 'LEFT_PARENTHESIS'),
(r'\)', 'RIGHT_PARENTHESIS'),
(r':', 'COLON'),
@@ -40,6 +40,7 @@ TOKENS = (
(r'/', 'SLASH'),
(r'\|', 'VBAR'),
(r'&', 'AMPER'),
(r'@', 'AT'),
(r'<', 'LESS'),
(r'>', 'GREATER'),
(r'=', 'EQUAL'),
@@ -61,6 +62,7 @@ TOKENS = (
(r'\*\*', 'DOUBLE_STAR'),
(r'\+=', 'PLUS_EQUAL'),
(r'-=', 'MINUS_EQUAL'),
(r'@=', 'AT_EQUAL'),
(r'\*=', 'STAR_EQUAL'),
(r'/=', 'SLASH_EQUAL'),
(r'%=', 'PERCENT_EQUAL'),
@@ -78,10 +80,13 @@ TOKENS = (
(r'(\s|\\\n|\\\r\n)+', 'SPACE'),
(r'["\'](.|\n|\r)*["\']', 'STRING'),
(r'[uU]["\'](.|\n|\r)*["\']', 'UNICODE_STRING'),
(r'[fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_STRING'),
(r'[rR]["\'](.|\n|\r)*["\']', 'RAW_STRING'),
(r'[bB]["\'](.|\n|\r)*["\']', 'BINARY_STRING'),
(r'[uU][rR]["\'](.|\n|\r)*["\']', 'UNICODE_RAW_STRING'),
(r'[bB][rR]["\'](.|\n|\r)*["\']', 'BINARY_RAW_STRING'),
(r'[fF][rR]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
(r'[rR][fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
)
 
 
docs/grammar.rst
View file @ e781669e
@@ -34,8 +34,23 @@ section**
Python 3.3 is the based grammar I've started diffing with, some of the grammar
differences marked as 3.3 are actually from older python version.
 
Current status
==============
Grammar diff has been done up to python 3.6.2.
Still, some stuff for the lexer are probably missing in this list like:
* fstrings
* adding _ in numbers
I need to got through all release notes to see that.
Done
====
Print function
~~~~~~~~~~~~~~
--------------
 
Python 3.3 or earlier
 
@@ -46,11 +61,96 @@ Already done since the start.
This is handle at the parser initialisation level, is activate or not the
print_function rule.
 
Ellipsis as atom (to replace 'pass')
------------------------------------
Python 3.3 or earlier
RedBaron: probably nothing to do (to be confirmed)
Baron: https://github.com/PyCQA/baron/pull/121
::
ADD '...' in atom rule
Matrix operator
---------------
Python 3.5
Baron: https://github.com/PyCQA/baron/pull/117
RedBaron: probably nothing to do (to be confirmed)
::
ADD '@' and '@=' to the lexer
ADD '@=' in augassign
ADD '@' in term
::
augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
'<<=' | '>>=' | '**=' | '//=')
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
f-strings
---------
Python 3.6
Baron: https://github.com/PyCQA/baron/pull/110
RedBaron: TODO
::
add f'' rf'' fr'' to the lexer
Underscore in numeric literals
------------------------------
Python 3.6
Baron: https://github.com/PyCQA/baron/pull/111
RedBaron: probably nothing to do (to be confirmed)
::
modify lexer to handle this
Nonlocal statement
------------------
Python 3.3 or earlier
Baron: https://github.com/PyCQA/baron/pull/112
RedBaron: TODO
.. image:: ./grammar_diff/nonlocal_statement.png
Action:
::
ADD 'nonlocal' to lexer
ADD 'nonlocal_stmt' to 'small_stmt'
ADD new rule:
nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
TODO
----
====
Ellipsis in from import
-----------------------
Python 3.3 or earlier
.. image:: ./grammar_diff/ellipsis_in_from_import.png
 
Typed arguments
~~~~~~~~~~~~~~~
---------------
 
Python 3.3 or earlier
 
@@ -96,7 +196,7 @@ Action:
 
 
Function return type
~~~~~~~~~~~~~~~~~~~~
--------------------
 
Python 3.3 or earlier
 
@@ -110,25 +210,8 @@ Action:
ADD ['->' test] to funcdef rule
funcdef: 'def' NAME parameters ['->' test] ':' suite
 
Nonlocal statement
~~~~~~~~~~~~~~~~~~
Python 3.3 or earlier
.. image:: ./grammar_diff/nonlocal_statement.png
Action:
::
ADD 'nonlocal' to lexer
ADD 'nonlocal_stmt' to 'small_stmt'
ADD new rule:
nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
Exec function
~~~~~~~~~~~~~
-------------
 
Python 3.3 or earlier
 
@@ -139,7 +222,7 @@ Like print_function but for 'exec'.
No one seems to be using that.
 
*var generalisation
~~~~~~~~~~~~~~~~~~~
-------------------
 
Python 3.3 or earlier
 
@@ -158,7 +241,7 @@ Python 3.3 or earlier
.. image:: ./grammar_diff/star_expr_in_expr_list.png
 
Raise from
~~~~~~~~~~
----------
 
Python 3.3 or earlier
 
@@ -177,15 +260,8 @@ Action:
# merge
raise_stmt: 'raise' [test [(',' test [',' test]] | 'from' test)]
 
Ellipsis in from import
~~~~~~~~~~~~~~~~~~~~~~~
Python 3.3 or earlier
.. image:: ./grammar_diff/ellipsis_in_from_import.png
New lambda grammar
~~~~~~~~~~~~~~~~~~
------------------
 
Python 3.3 or earlier
 
@@ -196,7 +272,7 @@ I have no idea on what to do with this one yet.
.. image:: ./grammar_diff/new_grammar_for_if_cond.png
 
Remove old list comprehension syntax
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
------------------------------------
 
Python 3.3 or earlier
 
@@ -209,7 +285,7 @@ actually used that one that?)
.. image:: ./grammar_diff/no_more_list_for_rule.png
 
False|True|None|... are now atoms in the grammar
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
------------------------------------------------
 
Python 3.3 or earlier
 
@@ -218,7 +294,7 @@ Do I need to do anything about that?
.. image:: ./grammar_diff/more_atoms.png
 
Inheritance in class definition uses arglist now
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
------------------------------------------------
 
Python 3.3 or earlier
 
@@ -227,12 +303,176 @@ I have no idea on why this is here but that's easy to change.
.. image:: ./grammar_diff/class_inherit_is_arglist_now.png
 
Yield From
~~~~~~~~~~
----------
 
Python 3.3 or earlier
 
.. image:: ./grammar_diff/yield_from.png
 
Async Funcdef
-------------
Python 3.5
Before:
::
decorated: decorators (classdef | funcdef)
After:
::
decorated: decorators (classdef | funcdef | async_funcdef)
async_funcdef: ASYNC funcdef
Await atom
----------
Python 3.5
Before:
::
power: atom trailer* ['**' factor]
After:
::
power: atom_expr ['**' factor]
atom_expr: [AWAIT] atom trailer*
Kwargs expressions
------------------
Python 3.5
Before:
::
dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
(test (comp_for | (',' test)* [','])) )
arglist: (argument ',')* (argument [',']
|'*' test (',' argument)* [',' '**' test]
|'**' test)
# The reason that keywords are test nodes instead of NAME is that using NAME
# results in an ambiguity. ast.c makes sure it's a NAME.
argument: test [comp_for] | test '=' test
After:
::
dictorsetmaker: ( ((test ':' test | '**' expr)
(comp_for | (',' (test ':' test | '**' expr))* [','])) |
((test | star_expr)
(comp_for | (',' (test | star_expr))* [','])) )
# can be simplified apparently
arglist: argument (',' argument)* [',']
# The reason that keywords are test nodes instead of NAME is that using NAME
# results in an ambiguity. ast.c makes sure it's a NAME.
# "test '=' test" is really "keyword '=' test", but we have no such token.
# These need to be in a single rule to avoid grammar that is ambiguous
# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
# we explicitly match '*' here, too, to give it proper precedence.
# Illegal combinations and orderings are blocked in ast.c:
# multiple (test comp_for) arguments are blocked; keyword unpackings
# that precede iterable unpackings are blocked; etc.
argument: ( test [comp_for] |
test '=' test |
'**' test |
'*' test )
Variables annotations
---------------------
Python 3.6
Before:
::
expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
('=' (yield_expr|testlist_star_expr))*)
After:
::
expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
('=' (yield_expr|testlist_star_expr))*)
annassign: ':' test ['=' test]
async for loop
--------------
Python 3.6
Before:
::
comp_for: 'for' exprlist 'in' or_test [comp_iter]
After:
::
comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
Refactoring in typedargslist ?
------------------------------
I think this is for asynchronous generator and comprehension:
* https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep525
* https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep530
Before:
::
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
| '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
| '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
After:
::
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
['*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
| '**' tfpdef [',']]]
| '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
| '**' tfpdef [','])
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
['*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']]]
| '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']
)
 
 
 
@@ -241,27 +481,27 @@ Python 3.3 or earlier
 
 
Nothing to do
-------------
=============
 
Those are things that have been removed from python3 grammar but we still need
to support (and we already do) so we don't have to do anything.
 
No more commat syntax in except close
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-------------------------------------
 
Python 3.3 or earlier
 
.. image:: ./grammar_diff/no_more_commat_in_execption_close.png
 
No more backquote syntax
~~~~~~~~~~~~~~~~~~~~~~~~
------------------------
 
Python 3.3 or earlier
 
.. image:: ./grammar_diff/no_more_backquote_syntax.png
 
No more '.' '.' '.' in the grammar
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
----------------------------------
 
Python 3.3 or earlier
 
grammar/baron_grammar 0 → 100644
View file @ e781669e
# Grammar for Baron
# this file is a REFERENCE on what grammar baron is supposed to be implementing
# to be a mixed between python2 and python3 grammar
# IT IS NOT USED BY BARON DIRECTLY
# Start symbols for the grammar:
# single_input is a single interactive statement;
# file_input is a module or sequence of commands read from an input file;
# eval_input is the input for the eval() and input() functions.
# NB: compound_stmt in single_input is followed by extra NEWLINE!
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
file_input: (NEWLINE | stmt)* ENDMARKER
eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
decorated: decorators (classdef | funcdef)
funcdef: 'def' NAME parameters ':' suite
parameters: '(' [varargslist] ')'
varargslist: ((fpdef ['=' test] ',')*
('*' NAME [',' '**' NAME] | '**' NAME) |
fpdef ['=' test] (',' fpdef ['=' test])* [','])
fpdef: NAME | '(' fplist ')'
fplist: fpdef (',' fpdef)* [',']
stmt: simple_stmt | compound_stmt
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt |
import_stmt | global_stmt | nonlocal_stmt | exec_stmt | assert_stmt)
expr_stmt: testlist (augassign (yield_expr|testlist) |
('=' (yield_expr|testlist))*)
augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
'<<=' | '>>=' | '**=' | '//=')
# For normal assignments, additional restrictions enforced by the interpreter
print_stmt: 'print' ( [ test (',' test)* [','] ] |
'>>' test [ (',' test)+ [','] ] )
del_stmt: 'del' exprlist
pass_stmt: 'pass'
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
break_stmt: 'break'
continue_stmt: 'continue'
return_stmt: 'return' [testlist]
yield_stmt: yield_expr
raise_stmt: 'raise' [test [',' test [',' test]]]
import_stmt: import_name | import_from
import_name: 'import' dotted_as_names
import_from: ('from' ('.'* dotted_name | '.'+)
'import' ('*' | '(' import_as_names ')' | import_as_names))
import_as_name: NAME ['as' NAME]
dotted_as_name: dotted_name ['as' NAME]
import_as_names: import_as_name (',' import_as_name)* [',']
dotted_as_names: dotted_as_name (',' dotted_as_name)*
dotted_name: NAME ('.' NAME)*
global_stmt: 'global' NAME (',' NAME)*
nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
try_stmt: ('try' ':' suite
((except_clause ':' suite)+
['else' ':' suite]
['finally' ':' suite] |
'finally' ':' suite))
with_stmt: 'with' with_item (',' with_item)* ':' suite
with_item: test ['as' expr]
# NB compile.c makes sure that the default except clause is last
except_clause: 'except' [test [('as' | ',') test]]
suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
# Backward compatibility cruft to support:
# [ x for x in lambda: True, lambda: False if x() ]
# even while also allowing:
# lambda x: 5 if x else 2
# (But not a mix of the two)
testlist_safe: old_test [(',' old_test)+ [',']]
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [varargslist] ':' old_test
test: or_test ['if' or_test 'else' test] | lambdef
lambdef: 'lambda' [varargslist] ':' test
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
comparison: expr (comp_op expr)*
comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
expr: xor_expr ('|' xor_expr)*
xor_expr: and_expr ('^' and_expr)*
and_expr: shift_expr ('&' shift_expr)*
shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [listmaker] ']' |
'{' [dictorsetmaker] '}' |
'`' testlist1 '`' |
NAME | NUMBER | STRING+ | '...')
testlist_comp: test ( comp_for | (',' test)* [','] )
listmaker: test ( list_for | (',' test)* [','] )
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
exprlist: expr (',' expr)* [',']
testlist: test (',' test)* [',']
dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
(test (comp_for | (',' test)* [','])) )
classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
arglist: (argument ',')* (argument [',']
|'*' test (',' argument)* [',' '**' test]
|'**' test)
# The reason that keywords are test nodes instead of NAME is that using NAME
# results in an ambiguity. ast.c makes sure it's a NAME.
argument: test [comp_for] | test '=' test
list_iter: list_for | list_if
list_for: 'for' exprlist 'in' testlist_safe [list_iter]
list_if: 'if' old_test [list_iter]
comp_iter: comp_for | comp_if
comp_for: 'for' exprlist 'in' or_test [comp_iter]
comp_if: 'if' old_test [comp_iter]
testlist1: test (',' test)*
# not used in grammar, but may appear in "node" passed from Parser to Compiler
encoding_decl: NAME
yield_expr: 'yield' [testlist]
grammar/reference_grammar_python3.6
View file @ e781669e
@@ -20,15 +20,15 @@ async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
 
parameters: '(' [typedargslist] ')'
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
'*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
['*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
| '**' tfpdef [',']]]
| '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
| '**' tfpdef [','])
tfpdef: NAME [':' test]
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
'*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']]]
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
['*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']]]
| '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']
)
@@ -146,4 +146,4 @@ comp_if: 'if' test_nocond [comp_iter]
encoding_decl: NAME
 
yield_expr: 'yield' [yield_arg]
yield_arg: 'from' test | testlist
\ No newline at end of file
yield_arg: 'from' test | testlist
tests/test_baron.py
View file @ e781669e
@@ -18,7 +18,7 @@ def test_error_parsing_error():
 
 
def test_error_unexpected_formatting():
with pytest.raises(UnExpectedFormattingToken):
with pytest.raises(ParsingError):
parse(" a\nb")
with pytest.raises(BaronError):
parse(" a\nb")
tests/test_dumper.py
View file @ e781669e
@@ -534,3 +534,7 @@ def test_try_import_after_colon():
 
def test_single_object():
assert baron.dumps({"type": "name", "value": "a"}) == "a"
def test_crash_issue_85():
check_dumps('d*e-1\n')
\ No newline at end of file
tests/test_formatting_grouper.py
View file @ e781669e
@@ -2148,6 +2148,7 @@ augmented_assignment_tokens = (
('SLASH_EQUAL', '/='),
('PERCENT_EQUAL', '%='),
('AMPER_EQUAL', '&='),
('AT_EQUAL', '@='),
('VBAR_EQUAL', '|='),
('CIRCUMFLEX_EQUAL', '^='),
('LEFT_SHIFT_EQUAL', '<<='),
@@ -2547,6 +2548,18 @@ def test_global_two():
])
 
 
def test_nonlocal():
"global a"
group([
('NONLOCAL', 'nonlocal'),
('SPACE', ' '),
('NAME', 'a'),
], [
('NONLOCAL', 'nonlocal', [], [('SPACE', ' ')]),
('NAME', 'a'),
])
def test_print():
"print"
group([
@@ -2889,7 +2902,7 @@ def test_strings():
"""
I don't this because python allow to write stuff like 'qsd' rb"qsd" u'pouet'
"""
for i in ('STRING', 'RAW_STRING', 'UNICODE_STRING', 'UNICODE_RAW_STRING', 'BINARY_STRING', 'BINARY_RAW_STRING'):
for i in ('STRING', 'RAW_STRING', 'UNICODE_STRING', 'INTERPOLATED_STRING', 'UNICODE_RAW_STRING', 'BINARY_STRING', 'INTERPOLATED_RAW_STRING', 'BINARY_RAW_STRING'):
group([
('SPACE', ' '),
(i, 'dummy'),
tests/test_grammator.py
View file @ e781669e
@@ -380,6 +380,17 @@ def test_file_input_simple_stmt_one_item_semicolon_space():
])
 
 
def test_ellipsis():
"..."
parse_simple([('DOT', '.'), ('DOT', '.'),('DOT', '.')], [
{
"type": "ellipsis",
"first_formatting": [],
"second_formatting": [],
}
])
def test_funcdef_stmt_indent():
"""
def a () :
tests/test_grammator_operators.py
View file @ e781669e
@@ -3502,6 +3502,222 @@ def test_arith_expr_substract_spaces_atomtrailers():
])
 
 
def test_arith_expr_at():
"a@b"
parse_simple([
('NAME', 'a'),
('MINUS', '@'),
('NAME', 'b'),
], [
{
"type": "binary_operator",
"value": '@',
"first": {
"type": "name",
"value": 'a',
},
"second": {
"type": "name",
"value": 'b'
},
"first_formatting": [],
"second_formatting": []
}
])
parse_simple([
('NAME', 'a'),
('MINUS', '@'),
('NAME', 'b'),
], [
{
"type": "binary_operator",
"value": '@',
"first": {
"type": "name",
"value": 'a'
},
"second": {
"type": "name",
"value": 'b',
},
"first_formatting": [],
"second_formatting": []
}
])
def test_arith_expr_at_first_space():
"a @b"
parse_simple([
('NAME', 'a'),
('MINUS', '@', [('SPACE', ' ')]),
('NAME', 'b'),
], [
{
"type": "binary_operator",
"value": '@',
"first": {
"type": "name",
"value": 'a',
},
"second": {
"type": "name",
"value": 'b'
},
"first_formatting": [{"type": "space", "value": " "}],
"second_formatting": []
}
])
parse_simple([
('NAME', 'a'),
('MINUS', '@', [('SPACE', ' ')]),
('NAME', 'b'),
], [
{
"type": "binary_operator",
"value": '@',
"first": {
"type": "name",
"value": 'a'
},
"second": {
"type": "name",
"value": 'b',
},
"first_formatting": [{"type": "space", "value": " "}],
"second_formatting": []
}
])
def test_arith_expr_at_second_space():
"a@ b"
parse_simple([
('NAME', 'a'),
('MINUS', '@', [], [('SPACE', ' ')]),
('NAME', 'b'),
], [
{
"type": "binary_operator",
"value": '@',
"first": {
"type": "name",
"value": 'a',
},
"second": {
"type": "name",
"value": 'b'
},
"first_formatting": [],
"second_formatting": [{"type": "space", "value": " "}]
}
])
parse_simple([
('NAME', 'a'),
('MINUS', '@', [], [('SPACE', ' ')]),
('NAME', 'b'),
], [
{
"type": "binary_operator",
"value": '@',
"first": {
"type": "name",
"value": 'a'
},
"second": {
"type": "name",
"value": 'b',
},
"first_formatting": [],
"second_formatting": [{"type": "space", "value": " "}]
}
])
def test_arith_expr_at_spaces():
"a @ b"
parse_simple([
('NAME', 'a'),
('MINUS', '@', [('SPACE', ' ')], [('SPACE', ' ')]),
('NAME', 'b'),
], [
{
"type": "binary_operator",
"value": '@',
"first": {
"type": "name",
"value": 'a',
},
"second": {
"type": "name",
"value": 'b'
},
"first_formatting": [{"type": "space", "value": " "}],
"second_formatting": [{"type": "space", "value": " "}]
}
])
parse_simple([
('NAME', 'a'),
('MINUS', '@', [('SPACE', ' ')], [('SPACE', ' ')]),
('NAME', 'b'),
], [
{
"type": "binary_operator",
"value": '@',
"first": {
"type": "name",
"value": 'a'
},
"second": {
"type": "name",
"value": 'b',
},
"first_formatting": [{"type": "space", "value": " "}],
"second_formatting": [{"type": "space", "value": " "}]
}
])
def test_arith_expr_at_spaces_atomtrailers():
"a.b - c"
parse_simple([
('NAME', 'a'),
('DOT', '.'),
('NAME', 'b'),
('MINUS', '-', [('SPACE', ' ')], [('SPACE', ' ')]),
('NAME', 'c'),
], [
{
"type": "binary_operator",
"value": '-',
"first": {
"type": "atomtrailers",
"value": [
{
"type": "name",
"value": "a",
},
{
"type": "dot",
"first_formatting": [],
"second_formatting": [],
},
{
"type": "name",
"value": "b",
}
],
},
"second": {
"type": "name",
"value": 'c',
},
"first_formatting": [{"type": "space", "value": " "}],
"second_formatting": [{"type": "space", "value": " "}]
}
])
def test_chained_add_substract():
"a+b-c"
parse_simple([
@@ -7658,6 +7874,7 @@ augmented_assignment_tokens = (
('SLASH_EQUAL', '/='),
('PERCENT_EQUAL', '%='),
('AMPER_EQUAL', '&='),
('AT_EQUAL', '@='),
('VBAR_EQUAL', '|='),
('CIRCUMFLEX_EQUAL', '^='),
('LEFT_SHIFT_EQUAL', '<<='),