Merge remote-tracking branch 'odcinek/pep-0515'

baron/tokenizer.py

@@ -20,14 +20,14 @@ TOKENS = (
     (r'\d+\.[jJ]', 'COMPLEX'),
     (r'\d+[jJ]', 'COMPLEX'),
     (r'\d+\.', 'FLOAT'),
-    (r'\d*\.\d+[lL]?', 'FLOAT'),
-    (r'\d+\.\d*[lL]?', 'FLOAT'),
+    (r'\d*[_\d]*\.[_\d]+[lL]?', 'FLOAT'),
+    (r'\d+[_\d]+\.[_\d]*[lL]?', 'FLOAT'),
     (r'\.', 'DOT'),
-    (r'[1-9]+\d*[lL]', 'LONG'),
-    (r'[1-9]+\d*', 'INT'),
-    (r'0[xX][\da-fA-F]+[lL]?', 'HEXA'),
-    (r'(0[oO][0-7]+)|(0[0-7]*)[lL]?', 'OCTA'),
-    (r'0[bB][01]+[lL]?', 'BINARY'),
+    (r'[1-9]+[_\d]*[lL]', 'LONG'),
+    (r'[1-9]+[_\d]*', 'INT'),
+    (r'0[xX][\d_a-fA-F]+[lL]?', 'HEXA'),
+    (r'(0[oO][0-7]+)|(0[0-7_]*)[lL]?', 'OCTA'),
+    (r'0[bB][01_]+[lL]?', 'BINARY'),
     (r'\(', 'LEFT_PARENTHESIS'),
     (r'\)', 'RIGHT_PARENTHESIS'),
     (r':', 'COLON'),

tests/test_tokenizer.py

@@ -51,6 +51,15 @@ def test_hexa():
     match('0x123ABCDL', 'HEXA')
+def test_grouped():
+    match('100_000', 'INT')
+    match('10_000_000.', 'FLOAT')
+    match('10_000_000.0', 'FLOAT')
+    match('0xCAFE_F00D', 'HEXA')
+    match('0o123_45', 'OCTA')
+    match('0b_0011_1111_0100_1110', 'BINARY')
+
+
 def test_octa():
     match('012345', 'OCTA')
     match('0o12345', 'OCTA')