Merge pull request #92 from PyCQA/fix_line_continuation

Fix line continuation

baron/baron.py

@@ -67,4 +67,10 @@ def parse(source_code, print_function=None):
 def tokenize(pouet, print_function=False):
-    return mark_indentation(inner_group(space_group(_tokenize(group(split(pouet)), print_function))))
+    splitted = split(pouet)
+    grouped = group(splitted)
+    print_tokenized = _tokenize(grouped, print_function)
+    space_grouped = space_group(print_tokenized)
+    inner_grouped = inner_group(space_grouped)
+    indentation_marked = mark_indentation(inner_grouped)
+    return indentation_marked

baron/inner_formatting_grouper.py

@@ -178,15 +178,4 @@ def group_generator(sequence):
                     debug_file_content += _append_to_debug_file_content(iterator.show_next())
                     current = append_to_token_after(current, [next(iterator)])
-
-        if current[0] == "SPACE":
-            debug_file_content = debug_file_content.split("\n")
-            debug_file_content = list(zip(range(1, len(debug_file_content) + 1), debug_file_content))
-            debug_file_content = debug_file_content[-3:]
-            debug_file_content = "\n".join(["%4s %s" % (x[0], x[1]) for x in debug_file_content])
-            debug_file_content += "<--- here"
-            debug_text = "Unexpected '%s' token:\n\n" % current[0].lower() + debug_file_content + "\n\n"
-            debug_text += "Should have been grouped on either %s (before) or %s (after) token." % (debug_previous_token, iterator.show_next())
-            raise UnExpectedFormattingToken(debug_text)
-
         yield current

tests/test_baron.py

@@ -18,7 +18,7 @@ def test_error_parsing_error():
 def test_error_unexpected_formatting():
-    with pytest.raises(UnExpectedFormattingToken):
+    with pytest.raises(ParsingError):
         parse("   a\nb")
     with pytest.raises(BaronError):
         parse("   a\nb")

tests/test_grouper.py

@@ -166,6 +166,10 @@ def test_space_endl_with_backslash():
     grouper_test(" 	 \\\npouet", [' 	 ', '\\', '\n', 'pouet'], [' 	 \\\n', 'pouet'])
+def test_number_with_backslash():
+    grouper_test("3\\\n", ['3', '\\', '\n'], ['3', '\\\n'])
+
+
 def test_regression():
     grouper_test("0x045e: ", ['0x045e', ':', ' '], ['0x045e', ':', ' '])
     grouper_test("180.\n", ['180', '.', '\n'], ['180.', '\n'])

tests/test_inner_formatting_grouper.py

@@ -163,3 +163,23 @@ def test_dict_one():
         ('NAME', 'a'),
         ('RIGHT_BRACKET', '}', [('ENDL', '\n')], []),
     ]
+
+
+def test_number_backslash():
+    assert group([
+        ('INT', '3'),
+        ('SPACE', '\\'),
+    ]) == [
+        ('INT', '3'),
+        ('SPACE', '\\'),
+    ]
+
+
+def test_number_backslash_newline():
+    assert group([
+        ('INT', '3'),
+        ('SPACE', '\\\n'),
+    ]) == [
+        ('INT', '3'),
+        ('SPACE', '\\\n'),
+    ]

tests/test_tokenizer.py

@@ -483,3 +483,7 @@ def test_exponant_complex():
     match("-.1E+1J", "FLOAT_EXPONANT_COMPLEX")
 # TODO 1.1e1j
+
+
+def test_backslash():
+    match("\\\n", "SPACE")