From 6cf12584ba8778f567417b382a177539d0cac7fc Mon Sep 17 00:00:00 2001
From: Monson Shao <holymonson@gmail.com>
Date: Mon, 27 Aug 2018 14:49:11 +0800
Subject: [PATCH 1/4] lib2to3: support non-ASCII identifiers

---
 Lib/lib2to3/pgen2/tokenize.py    | 11 +++++------
 Lib/lib2to3/tests/test_parser.py |  6 ++++++
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index bf83ab8c7d9d2e..0160dc316ad827 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -56,7 +56,7 @@ def _combinations(*l):
 Whitespace = r'[ \f\t]*'
 Comment = r'#[^\r\n]*'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
-Name = r'[a-zA-Z_]\w*'
+Name = r'\w(?<!\d)\w*'
 
 Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
 Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
@@ -107,8 +107,8 @@ def _combinations(*l):
 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
-tokenprog, pseudoprog, single3prog, double3prog = list(map(
-    re.compile, (Token, PseudoToken, Single3, Double3)))
+nameprog, tokenprog, pseudoprog, single3prog, double3prog = map(
+    re.compile, (Name, Token, PseudoToken, Single3, Double3))
 
 _strprefixes = (
     _combinations('r', 'R', 'f', 'F') |
@@ -349,7 +349,6 @@ def generate_tokens(readline):
     logical line; continuation lines are included.
     """
     lnum = parenlev = continued = 0
-    namechars, numchars = string.ascii_letters + '_', '0123456789'
     contstr, needcont = '', 0
     contline = None
     indents = [0]
@@ -451,7 +450,7 @@ def generate_tokens(readline):
                 spos, epos, pos = (lnum, start), (lnum, end), end
                 token, initial = line[start:end], line[start]
 
-                if initial in numchars or \
+                if initial in string.digits or \
                    (initial == '.' and token != '.'):      # ordinary number
                     yield (NUMBER, token, spos, epos, line)
                 elif initial in '\r\n':
@@ -501,7 +500,7 @@ def generate_tokens(readline):
                             yield stashed
                             stashed = None
                         yield (STRING, token, spos, epos, line)
-                elif initial in namechars:                 # ordinary name
+                elif nameprog.match(token):                # ordinary name
                     if token in ('async', 'await'):
                         if async_def:
                             yield (ASYNC if token == 'async' else AWAIT,
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 7eb9afac46e0c7..a0e61694cb289f 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -529,6 +529,12 @@ def test_4(self):
         self.validate("""x = {2, 3, 4,}""")
 
 
+# Adapted from Python 3's Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers
+class TestIdentfier(GrammarTest):
+    def test_non_ascii_identifiers(self):
+        self.validate("Örter = 'places'\ngrün = 'green'")
+
+
 class TestNumericLiterals(GrammarTest):
     def test_new_octal_notation(self):
         self.validate("""0o7777777777777""")

From 07747382d82149ac459b826f418485af2dc7f43e Mon Sep 17 00:00:00 2001
From: Monson Shao <holymonson@gmail.com>
Date: Mon, 27 Aug 2018 16:09:16 +0800
Subject: [PATCH 2/4] add news

---
 .../NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst

diff --git a/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst b/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst
new file mode 100644
index 00000000000000..c1a5c6adf08a6a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst
@@ -0,0 +1 @@
+Add non-ASCII identifiers support in :mod:`lib2to3.pgen2.tokenize`.

From fb3a039664a63d12a6732d012a65899e4aadc9dd Mon Sep 17 00:00:00 2001
From: Monson Shao <holymonson@gmail.com>
Date: Mon, 27 Aug 2018 20:11:00 +0800
Subject: [PATCH 3/4] add more test

---
 Lib/lib2to3/tests/test_parser.py                            | 6 +++++-
 .../next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst   | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index a0e61694cb289f..829e5a72924aa4 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -529,10 +529,14 @@ def test_4(self):
         self.validate("""x = {2, 3, 4,}""")
 
 
-# Adapted from Python 3's Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers
+# Adapted from Python 3's Lib/test/test_unicode_identifiers.py and
+# Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers
 class TestIdentfier(GrammarTest):
     def test_non_ascii_identifiers(self):
         self.validate("Örter = 'places'\ngrün = 'green'")
+        self.validate("蟒 = a蟒 = 锦蛇 = 1")
+        self.validate("µ = aµ = µµ = 1")
+        self.validate("𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = a_𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1")
 
 
 class TestNumericLiterals(GrammarTest):
diff --git a/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst b/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst
index c1a5c6adf08a6a..7ace7ba8218c00 100644
--- a/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst
+++ b/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst
@@ -1 +1 @@
-Add non-ASCII identifiers support in :mod:`lib2to3.pgen2.tokenize`.
+Fix parsing non-ASCII identifiers in :mod:`lib2to3.pgen2.tokenize` (PEP 3131).

From 37d8770d537f0b55e867c8383619e157a6daafe3 Mon Sep 17 00:00:00 2001
From: Monson Shao <holymonson@gmail.com>
Date: Sat, 15 Sep 2018 10:53:05 +0800
Subject: [PATCH 4/4] imitate tokenize.py

---
 Lib/lib2to3/pgen2/tokenize.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 0160dc316ad827..c07b34f7c6452d 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -56,7 +56,7 @@ def _combinations(*l):
 Whitespace = r'[ \f\t]*'
 Comment = r'#[^\r\n]*'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
-Name = r'\w(?<!\d)\w*'
+Name = r'\w+'
 
 Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
 Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
@@ -107,8 +107,8 @@ def _combinations(*l):
 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
-nameprog, tokenprog, pseudoprog, single3prog, double3prog = map(
-    re.compile, (Name, Token, PseudoToken, Single3, Double3))
+tokenprog, pseudoprog, single3prog, double3prog = map(
+    re.compile, (Token, PseudoToken, Single3, Double3))
 
 _strprefixes = (
     _combinations('r', 'R', 'f', 'F') |
@@ -500,7 +500,7 @@ def generate_tokens(readline):
                             yield stashed
                             stashed = None
                         yield (STRING, token, spos, epos, line)
-                elif nameprog.match(token):                # ordinary name
+                elif initial.isidentifier():               # ordinary name
                     if token in ('async', 'await'):
                         if async_def:
                             yield (ASYNC if token == 'async' else AWAIT,