Merge branch 'pep-0498'

baron/formatting_grouper.py

@@ -67,6 +67,8 @@ BOTH = (
 STRING = (
     "STRING",
     "RAW_STRING",
+    "INTERPOLATED_STRING",
+    "INTERPOLATED_RAW_STRING",
     "UNICODE_STRING",
     "UNICODE_RAW_STRING",
     "BINARY_STRING",

baron/grammator.py

@@ -655,10 +655,12 @@ def generate_parse(print_function):
     # TODO tests those other kind of strings
     @pg.production("string : STRING")
     @pg.production("string : RAW_STRING")
+    @pg.production("string : INTERPOLATED_STRING")
     @pg.production("string : UNICODE_STRING")
     @pg.production("string : BINARY_STRING")
     @pg.production("string : UNICODE_RAW_STRING")
     @pg.production("string : BINARY_RAW_STRING")
+    @pg.production("string : INTERPOLATED_RAW_STRING")
     def string(pack):
         (string_,) = pack
         return [{

baron/grouper.py

@@ -52,9 +52,9 @@ def group_generator(sequence):
             current += next(iterator)
         if current in to_group_keys and matching_found(to_group, current, iterator.show_next()):
             current += next(iterator)
-        if current in list('uUrRbB') and str(iterator.show_next()).startswith(('"', "'")):
+        if current in list('uUfFrRbB') and str(iterator.show_next()).startswith(('"', "'")):
             current += next(iterator)
-        if str(current).lower() in ["ur", "br"] and str(iterator.show_next()).startswith(('"', "'")):
+        if str(current).lower() in ["ur", "br", "fr"] and str(iterator.show_next()).startswith(('"', "'")):
             current += next(iterator)
         if any([re.match(x, current) for x in (r'^\d+[eE]$', r'^\d+\.\d*[eE]$', r'^\.\d+[eE]$')]):
             current += next(iterator)

baron/inner_formatting_grouper.py

@@ -35,6 +35,8 @@ GROUP_ON = (
     # TODO test everything bellow
     "STRING",
     "RAW_STRING",
+    "INTERPOLATED_STRING",
+    "INTERPOLATED_RAW_STRING",
     "BINARY_STRING",
     "BINARY_RAW_STRING",
     "UNICODE_STRING",

baron/tokenizer.py

@@ -80,10 +80,12 @@ TOKENS = (
     (r'(\s|\\\n|\\\r\n)+', 'SPACE'),
     (r'["\'](.|\n|\r)*["\']', 'STRING'),
     (r'[uU]["\'](.|\n|\r)*["\']', 'UNICODE_STRING'),
+    (r'[fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_STRING'),
     (r'[rR]["\'](.|\n|\r)*["\']', 'RAW_STRING'),
     (r'[bB]["\'](.|\n|\r)*["\']', 'BINARY_STRING'),
     (r'[uU][rR]["\'](.|\n|\r)*["\']', 'UNICODE_RAW_STRING'),
     (r'[bB][rR]["\'](.|\n|\r)*["\']', 'BINARY_RAW_STRING'),
+    (r'[fF][rR]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
 )

tests/test_formatting_grouper.py

@@ -2890,7 +2890,7 @@ def test_strings():
     """
     I don't this because python allow to write stuff like 'qsd' rb"qsd" u'pouet'
     """
-    for i in ('STRING', 'RAW_STRING', 'UNICODE_STRING', 'UNICODE_RAW_STRING', 'BINARY_STRING', 'BINARY_RAW_STRING'):
+    for i in ('STRING', 'RAW_STRING', 'UNICODE_STRING', 'INTERPOLATED_STRING', 'UNICODE_RAW_STRING', 'BINARY_STRING', 'INTERPOLATED_RAW_STRING', 'BINARY_RAW_STRING'):
         group([
             ('SPACE', ' '),
             (i, 'dummy'),

tests/test_tokenizer.py

@@ -309,6 +309,27 @@ def test_unicode_string():
     match("U'''pouet pouet'''", "UNICODE_STRING")
+def test_interpolated_string():
+    match("f'He said his name is {name!r}.'", 'INTERPOLATED_STRING')
+    match("f'The value is {value}.'", 'INTERPOLATED_STRING')
+    match('F"He said his name is {name!r}."', 'INTERPOLATED_STRING')
+    match('f"The value is {value}."', 'INTERPOLATED_STRING')
+    match("F'{date} was on a {date:%A}'", 'INTERPOLATED_STRING')
+    match("f'a={d[\"a\"]}'", 'INTERPOLATED_STRING')
+    match("F'{(lambda x: x*2)(3)}'", 'INTERPOLATED_STRING')
+
+
+def test_interpolated_raw_string():
+    match("fr'He said his name is {name!r}.'", 'INTERPOLATED_RAW_STRING')
+    match("fr'The value is {value}.'", 'INTERPOLATED_RAW_STRING')
+    match('Fr"He said his name is {name!r}."', 'INTERPOLATED_RAW_STRING')
+    match('fR"The value is {value}."', 'INTERPOLATED_RAW_STRING')
+    match("FR'{date} was on a {date:%A}'", 'INTERPOLATED_RAW_STRING')
+    match("fr'a={d[\"a\"]}'", 'INTERPOLATED_RAW_STRING')
+    match("FR'{(lambda x: x*2)(3)}'", 'INTERPOLATED_RAW_STRING')
+    match("FR'{(lambda x: x*2)(3)}'", 'INTERPOLATED_RAW_STRING')
+
+
 def test_raw_string():
     match('r"pouet pouet"', 'RAW_STRING')
     match("r'pouet pouet'", "RAW_STRING")