From ca6cd26cdc8299fafdd0c35ca3deadfa62197e7a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Nov 2017 01:00:49 +0200 Subject: [PATCH 01/13] bpo-10544: Disallow "yield" in comprehensions and generator expressions. --- Doc/reference/expressions.rst | 9 +++++++++ Doc/whatsnew/3.7.rst | 6 ++++++ Lib/test/test_generators.py | 8 +++----- Lib/test/test_grammar.py | 15 +++++++++++++++ .../2017-11-26-00-59-22.bpo-10544.fHOM3V.rst | 2 ++ Python/symtable.c | 12 +++++++++++- 6 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 1cff8a52df959d..7cbb81ea61ad48 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -186,6 +186,9 @@ each time the innermost block is reached. Note that the comprehension is executed in a separate scope, so names assigned to in the target list don't "leak" into the enclosing scope. +Yield expressions are not allowed in comprehensions +except the expression for the outermost iterable. + Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. A comprehension in an :keyword:`async def` function may consist of either a @@ -326,6 +329,9 @@ range(10) for y in bar(x))``. The parentheses can be omitted on calls with only one argument. See section :ref:`calls` for details. +Yield expressions are not allowed in generator expressions +except the expression for the outermost iterable. + If a generator expression contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an :dfn:`asynchronous generator expression`. An asynchronous generator @@ -364,6 +370,9 @@ coroutine function to be an asynchronous generator. For example:: async def agen(): # defines an asynchronous generator function (PEP 525) yield 123 +Yield expressions are not allowed in comprehensions and generator expressions +except the expression for the outermost iterable. + Generator functions are described below, while asynchronous generator functions are described separately in section :ref:`asynchronous-generator-functions`. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 514c3c293c080c..1de9c98d433e33 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -673,6 +673,12 @@ Changes in Python behavior parentheses can be omitted only on calls. (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.) +* Yield expressions now are not allowed in comprehensions and generator + expressions except the expression for the outermost iterable since + comprehensions and generator expressions are implemented using + an internal function. + (Contributed by Serhiy Storchaka in :issue:`10544`.) + Changes in the Python API ------------------------- diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 7eac9d076be462..92118ad7acf067 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1834,7 +1834,9 @@ def printsolution(self, x): An obscene abuse of a yield expression within a generator expression: >>> list((yield 21) for i in range(4)) -[21, None, 21, None, 21, None, 21, None] +Traceback (most recent call last): + ... +SyntaxError: 'yield' inside generator expression And a more sane, but still weird usage: @@ -2106,10 +2108,6 @@ def printsolution(self, x): >>> type(f()) ->>> def f(): x=(i for i in (yield) if (yield)) ->>> type(f()) - - >>> def f(d): d[(yield "a")] = d[(yield "b")] = 27 >>> data = [1,2] >>> g = f(data) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 65e26bfd383823..9cb20f2c7e7637 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -841,6 +841,21 @@ def g(): f((yield from ()), 1) # Check annotation refleak on SyntaxError check_syntax_error(self, "def g(a:(yield)): pass") + def test_yield_in_comprehensions(self): + # Check yield in comprehensions + def g(): [x for x in [(yield 1)]] + def g(): [x for x in [(yield from ())]] + check_syntax_error(self, "def g(): [(yield x) for x in ()]") + check_syntax_error(self, "def g(): [x for x in () if not (yield x)]") + check_syntax_error(self, "def g(): [y for x in () for y in [(yield x)]]") + check_syntax_error(self, "def g(): {(yield x) for x in ()}") + check_syntax_error(self, "def g(): {(yield x): x for x in ()}") + check_syntax_error(self, "def g(): {x: (yield x) for x in ()}") + check_syntax_error(self, "def g(): ((yield x) for x in ())") + check_syntax_error(self, "def g(): [(yield from x) for x in ()]") + check_syntax_error(self, "class C: [(yield x) for x in ()]") + check_syntax_error(self, "[(yield x) for x in ()]") + def test_raise(self): # 'raise' test [',' test] try: raise RuntimeError('just testing') diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst new file mode 100644 index 00000000000000..812e7b42f523aa --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst @@ -0,0 +1,2 @@ +Yield expressions now are disallowed in comprehensions and generator +expressions except the expression for the outermost iterable. diff --git a/Python/symtable.c b/Python/symtable.c index 55815c91cc9cab..3ef9ed16815d4b 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1734,7 +1734,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, e->lineno, e->col_offset)) { return 0; } - st->st_cur->ste_generator = is_generator; if (outermost->is_async) { st->st_cur->ste_coroutine = 1; } @@ -1754,6 +1753,17 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, if (value) VISIT(st, expr, value); VISIT(st, expr, elt); + if (st->st_cur->ste_generator) { + PyErr_SetString(PyExc_SyntaxError, + (e->kind == ListComp_kind) ? "'yield' inside list comprehension" : + (e->kind == SetComp_kind) ? "'yield' inside set comprehension" : + (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" : + "'yield' inside generator expression"); + PyErr_SyntaxLocationObject(st->st_filename, + st->st_cur->ste_lineno, + st->st_cur->ste_col_offset); + } + st->st_cur->ste_generator = is_generator; return symtable_exit_block(st, (void *)e); } From 5e133a1f043cca01ab0c2b7b1bb9c210fd6c61e7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2017 09:22:00 +0200 Subject: [PATCH 02/13] Emit a warning istead of an error. --- Doc/whatsnew/3.7.rst | 12 +++--- Lib/test/test_generators.py | 10 +---- Lib/test/test_grammar.py | 40 ++++++++++++++----- ... 2017-11-27-08-37-34.bpo-10544.07nioT.rst} | 2 +- Python/symtable.c | 27 ++++++++++--- 5 files changed, 60 insertions(+), 31 deletions(-) rename Misc/NEWS.d/next/Core and Builtins/{2017-11-26-00-59-22.bpo-10544.fHOM3V.rst => 2017-11-27-08-37-34.bpo-10544.07nioT.rst} (51%) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 1de9c98d433e33..87e92bbbaea4cf 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -556,6 +556,12 @@ Other CPython Implementation Changes Deprecated ========== +* Yield expressions now are not deprecated in comprehensions and generator + expressions except the expression for the outermost iterable since + comprehensions and generator expressions are implemented using + an internal function. + (Contributed by Serhiy Storchaka in :issue:`10544`.) + - Function :c:func:`PySlice_GetIndicesEx` is deprecated and replaced with a macro if ``Py_LIMITED_API`` is not set or set to the value between ``0x03050400`` and ``0x03060000`` (not including) or ``0x03060100`` or @@ -673,12 +679,6 @@ Changes in Python behavior parentheses can be omitted only on calls. (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.) -* Yield expressions now are not allowed in comprehensions and generator - expressions except the expression for the outermost iterable since - comprehensions and generator expressions are implemented using - an internal function. - (Contributed by Serhiy Storchaka in :issue:`10544`.) - Changes in the Python API ------------------------- diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 92118ad7acf067..f88c762581da61 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1830,15 +1830,7 @@ def printsolution(self, x): [None] - -An obscene abuse of a yield expression within a generator expression: - ->>> list((yield 21) for i in range(4)) -Traceback (most recent call last): - ... -SyntaxError: 'yield' inside generator expression - -And a more sane, but still weird usage: +Yield is allowed only in the outermost iterable in generator expression: >>> def f(): list(i for i in [(yield 26)]) >>> type(f()) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 9cb20f2c7e7637..823315f8cd0dfc 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -845,16 +845,36 @@ def test_yield_in_comprehensions(self): # Check yield in comprehensions def g(): [x for x in [(yield 1)]] def g(): [x for x in [(yield from ())]] - check_syntax_error(self, "def g(): [(yield x) for x in ()]") - check_syntax_error(self, "def g(): [x for x in () if not (yield x)]") - check_syntax_error(self, "def g(): [y for x in () for y in [(yield x)]]") - check_syntax_error(self, "def g(): {(yield x) for x in ()}") - check_syntax_error(self, "def g(): {(yield x): x for x in ()}") - check_syntax_error(self, "def g(): {x: (yield x) for x in ()}") - check_syntax_error(self, "def g(): ((yield x) for x in ())") - check_syntax_error(self, "def g(): [(yield from x) for x in ()]") - check_syntax_error(self, "class C: [(yield x) for x in ()]") - check_syntax_error(self, "[(yield x) for x in ()]") + + def check(code, warntext): + with self.assertWarnsRegex(DeprecationWarning, warntext): + compile(code, '', 'exec') + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings('error', category=DeprecationWarning) + with self.assertRaisesRegex(SyntaxError, warntext): + compile(code, '', 'exec') + + check("def g(): [(yield x) for x in ()]", + "'yield' inside list comprehension") + check("def g(): [x for x in () if not (yield x)]", + "'yield' inside list comprehension") + check("def g(): [y for x in () for y in [(yield x)]]", + "'yield' inside list comprehension") + check("def g(): {(yield x) for x in ()}", + "'yield' inside set comprehension") + check("def g(): {(yield x): x for x in ()}", + "'yield' inside dict comprehension") + check("def g(): {x: (yield x) for x in ()}", + "'yield' inside dict comprehension") + check("def g(): ((yield x) for x in ())", + "'yield' inside generator expression") + check("def g(): [(yield from x) for x in ()]", + "'yield' inside list comprehension") + check("class C: [(yield x) for x in ()]", + "'yield' inside list comprehension") + check("[(yield x) for x in ()]", + "'yield' inside list comprehension") def test_raise(self): # 'raise' test [',' test] diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst similarity index 51% rename from Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst rename to Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst index 812e7b42f523aa..4f966704fdb284 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst @@ -1,2 +1,2 @@ -Yield expressions now are disallowed in comprehensions and generator +Yield expressions now are deprecated in comprehensions and generator expressions except the expression for the outermost iterable. diff --git a/Python/symtable.c b/Python/symtable.c index 3ef9ed16815d4b..82c9943459ba37 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1754,16 +1754,33 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, VISIT(st, expr, value); VISIT(st, expr, elt); if (st->st_cur->ste_generator) { - PyErr_SetString(PyExc_SyntaxError, + PyObject *msg = PyUnicode_FromString( (e->kind == ListComp_kind) ? "'yield' inside list comprehension" : (e->kind == SetComp_kind) ? "'yield' inside set comprehension" : (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" : "'yield' inside generator expression"); - PyErr_SyntaxLocationObject(st->st_filename, - st->st_cur->ste_lineno, - st->st_cur->ste_col_offset); + if (msg == NULL) { + symtable_exit_block(st, (void *)e); + return 0; + } + if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning, + msg, st->st_filename, st->st_cur->ste_lineno, + NULL, NULL) == -1) + { + if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { + /* Turn a DeprecationWarning into a SyntaxError. */ + PyErr_SetObject(PyExc_SyntaxError, msg); + PyErr_SyntaxLocationObject(st->st_filename, + st->st_cur->ste_lineno, + st->st_cur->ste_col_offset); + } + Py_DECREF(msg); + symtable_exit_block(st, (void *)e); + return 0; + } + Py_DECREF(msg); } - st->st_cur->ste_generator = is_generator; + st->st_cur->ste_generator |= is_generator; return symtable_exit_block(st, (void *)e); } From bad2b059cb8850efa7935a0bcb763549fbaaa7dc Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 Nov 2017 20:04:05 +1000 Subject: [PATCH 03/13] Reword language reference updates - updated the comprehension scoping definition to use some of the words from the generator expression definition - added appropriate versionadded/changed directives - --- Doc/reference/expressions.rst | 47 +++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 7cbb81ea61ad48..746820bcb2ece8 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -183,11 +183,20 @@ by considering each of the :keyword:`for` or :keyword:`if` clauses a block, nesting from left to right, and evaluating the expression to produce an element each time the innermost block is reached. -Note that the comprehension is executed in a separate scope, so names assigned -to in the target list don't "leak" into the enclosing scope. +However, aside from the leftmost :keyword:`for` clause, the comprehension is +executed in a separate implicitly nested scope. This ensures that names +assigned to in the target list don't "leak" into the enclosing scope. -Yield expressions are not allowed in comprehensions -except the expression for the outermost iterable. +The leftmost :keyword:`for` clause is evaluated directly in the enclosing scope +and then passed as an argument to the implictly nested scope. Subsequent +:keyword:`for` clauses cannot be evaluated in the enclosing scope since they +may depend on the previous :keyword:`for` loop. For example: +``[x*y for x in range(10) for y in bar(x)]``. + +To ensure the comprehension always results in a container of the appropriate +type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly +nested scope (in Python 3.7, such expressions emit :exc:`DeprecationWarning` +when compiled, in Python 3.8+ they will emit `:exc:`SyntaxError`). Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. @@ -201,6 +210,12 @@ or :keyword:`await` expressions it is called an suspend the execution of the coroutine function in which it appears. See also :pep:`530`. +.. versionadded:: 3.6 + Asynchronous comprehensions were introduced + +.. versionchanged:: 3.7 + ``yield`` and ``yield from`` deprecated in the implicitly nested scope + .. _lists: List displays @@ -329,8 +344,11 @@ range(10) for y in bar(x))``. The parentheses can be omitted on calls with only one argument. See section :ref:`calls` for details. -Yield expressions are not allowed in generator expressions -except the expression for the outermost iterable. +To avoid interfering with the expected operation of the generator expression +itself, ``yield`` and ``yield from`` expressions are prohibited in the +implicitly defined generator (in Python 3.7, such expressions emit +:exc:`DeprecationWarning` when compiled, in Python 3.8+ they will emit +`:exc:`SyntaxError`). If a generator expression contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an @@ -338,11 +356,17 @@ clauses or :keyword:`await` expressions it is called an expression returns a new asynchronous generator object, which is an asynchronous iterator (see :ref:`async-iterators`). +.. versionadded:: 3.6 + Asynchronous generator expressions were introduced + .. versionchanged:: 3.7 Prior to Python 3.7, asynchronous generator expressions could only appear in :keyword:`async def` coroutines. Starting with 3.7, any function can use asynchronous generator expressions. +.. versionchanged:: 3.7 + ``yield`` and ``yield from`` deprecated in the implicitly nested scope + .. _yieldexpr: Yield expressions @@ -370,8 +394,15 @@ coroutine function to be an asynchronous generator. For example:: async def agen(): # defines an asynchronous generator function (PEP 525) yield 123 -Yield expressions are not allowed in comprehensions and generator expressions -except the expression for the outermost iterable. +Due to their side effects on the containing scope, ``yield`` expressions +are not permitted as part of the implicitly defined scopes used to +implement comprehensions and generator expressions (in Python 3.7, such +expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+ +they will emit `:exc:`SyntaxError`).. + +.. versionchanged:: 3.7 + Yield expressions deprecated in the implicitly nested scopes used to + implement comprehensions and generator expressions Generator functions are described below, while asynchronous generator functions are described separately in section From 109fd322bc4e82053dc1227638724ba9e823e1ba Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 Nov 2017 20:06:18 +1000 Subject: [PATCH 04/13] Fix typo in markup changes --- Doc/reference/expressions.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 746820bcb2ece8..6cb326b66487bd 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -196,7 +196,7 @@ may depend on the previous :keyword:`for` loop. For example: To ensure the comprehension always results in a container of the appropriate type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly nested scope (in Python 3.7, such expressions emit :exc:`DeprecationWarning` -when compiled, in Python 3.8+ they will emit `:exc:`SyntaxError`). +when compiled, in Python 3.8+ they will emit :exc:`SyntaxError`). Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. @@ -348,7 +348,7 @@ To avoid interfering with the expected operation of the generator expression itself, ``yield`` and ``yield from`` expressions are prohibited in the implicitly defined generator (in Python 3.7, such expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+ they will emit -`:exc:`SyntaxError`). +:exc:`SyntaxError`). If a generator expression contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an @@ -398,7 +398,7 @@ Due to their side effects on the containing scope, ``yield`` expressions are not permitted as part of the implicitly defined scopes used to implement comprehensions and generator expressions (in Python 3.7, such expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+ -they will emit `:exc:`SyntaxError`).. +they will emit :exc:`SyntaxError`).. .. versionchanged:: 3.7 Yield expressions deprecated in the implicitly nested scopes used to From e89b0c5dbe5c87856b1580124429cb4696a175d9 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 Nov 2017 20:12:52 +1000 Subject: [PATCH 05/13] Reword What's New notice --- Doc/whatsnew/3.7.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 87e92bbbaea4cf..3eea10543871e3 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -556,10 +556,13 @@ Other CPython Implementation Changes Deprecated ========== -* Yield expressions now are not deprecated in comprehensions and generator - expressions except the expression for the outermost iterable since - comprehensions and generator expressions are implemented using - an internal function. +* Yield expressions (both ``yield`` and ``yield from`` clauses) are now deprecated + in comprehensions and generator expressions (aside from the outermost iterable + defined in the leftmost :keyword`for` clause). This ensures that comprehensions + always immediately return a container of the appropriate type (rather than + potentially returning a :term:`generator-iterator` object), while generator + expressions won't attempt to interleave their implicit output with the output + from any explicit yield expressions. (Contributed by Serhiy Storchaka in :issue:`10544`.) - Function :c:func:`PySlice_GetIndicesEx` is deprecated and replaced with From 802a362e43586d93c92d770e39c415fc43725dad Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 Nov 2017 20:16:11 +1000 Subject: [PATCH 06/13] Reword NEWS entry --- .../2017-11-27-08-37-34.bpo-10544.07nioT.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst index 4f966704fdb284..555c94ebaee23a 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst @@ -1,2 +1,3 @@ -Yield expressions now are deprecated in comprehensions and generator -expressions except the expression for the outermost iterable. +Yield expressions are now deprecated in comprehensions and generator +expressions. They are still permitted in the definition of the outermost +iterable, as that is evaluated directly in the enclosing scope. From 5b9498d77aa8f8760ea13b55bd968f917ac4c425 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 Nov 2017 20:19:50 +1000 Subject: [PATCH 07/13] Describe specific consequences in What's New entry --- Doc/whatsnew/3.7.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 3eea10543871e3..a418ae3749930d 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -563,7 +563,10 @@ Deprecated potentially returning a :term:`generator-iterator` object), while generator expressions won't attempt to interleave their implicit output with the output from any explicit yield expressions. - (Contributed by Serhiy Storchaka in :issue:`10544`.) + + In Python 3.7, such expressions emit :exc:`DeprecationWarning` when compiled, + in Python 3.8+ they will emit :exc:`SyntaxError`. (Contributed by Serhiy + Storchaka in :issue:`10544`.) - Function :c:func:`PySlice_GetIndicesEx` is deprecated and replaced with a macro if ``Py_LIMITED_API`` is not set or set to the value between From afde4b58fe5bec77dbffb8be9a71c028fc21b4ab Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 Nov 2017 20:33:33 +1000 Subject: [PATCH 08/13] Remove trailing whitespace --- Doc/whatsnew/3.7.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index a418ae3749930d..fa2aa91006a8f9 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -563,7 +563,7 @@ Deprecated potentially returning a :term:`generator-iterator` object), while generator expressions won't attempt to interleave their implicit output with the output from any explicit yield expressions. - + In Python 3.7, such expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+ they will emit :exc:`SyntaxError`. (Contributed by Serhiy Storchaka in :issue:`10544`.) From c63cfb810a5a05899e593485e963ed7fa00f6e98 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 Nov 2017 20:41:31 +1000 Subject: [PATCH 09/13] Incorporate Serhiy's review comments --- Doc/reference/expressions.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 6cb326b66487bd..9fdc5c14c61257 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -191,7 +191,7 @@ The leftmost :keyword:`for` clause is evaluated directly in the enclosing scope and then passed as an argument to the implictly nested scope. Subsequent :keyword:`for` clauses cannot be evaluated in the enclosing scope since they may depend on the previous :keyword:`for` loop. For example: -``[x*y for x in range(10) for y in bar(x)]``. +``[x*y for x in range(10) for y in range(x, x+10)]``. To ensure the comprehension always results in a container of the appropriate type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly @@ -211,9 +211,9 @@ suspend the execution of the coroutine function in which it appears. See also :pep:`530`. .. versionadded:: 3.6 - Asynchronous comprehensions were introduced + Asynchronous comprehensions were introduced. -.. versionchanged:: 3.7 +.. deprecated:: 3.7 ``yield`` and ``yield from`` deprecated in the implicitly nested scope .. _lists: @@ -339,7 +339,7 @@ immediately evaluated, so that an error produced by it can be seen before any other possible error in the code that handles the generator expression. Subsequent :keyword:`for` clauses cannot be evaluated immediately since they may depend on the previous :keyword:`for` loop. For example: ``(x*y for x in -range(10) for y in bar(x))``. +range(10) for y in range(x, x+10))``. The parentheses can be omitted on calls with only one argument. See section :ref:`calls` for details. @@ -357,14 +357,14 @@ expression returns a new asynchronous generator object, which is an asynchronous iterator (see :ref:`async-iterators`). .. versionadded:: 3.6 - Asynchronous generator expressions were introduced + Asynchronous generator expressions were introduced. .. versionchanged:: 3.7 Prior to Python 3.7, asynchronous generator expressions could only appear in :keyword:`async def` coroutines. Starting with 3.7, any function can use asynchronous generator expressions. -.. versionchanged:: 3.7 +.. deprecated:: 3.7 ``yield`` and ``yield from`` deprecated in the implicitly nested scope .. _yieldexpr: @@ -400,7 +400,7 @@ implement comprehensions and generator expressions (in Python 3.7, such expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+ they will emit :exc:`SyntaxError`).. -.. versionchanged:: 3.7 +.. deprecated:: 3.7 Yield expressions deprecated in the implicitly nested scopes used to implement comprehensions and generator expressions From 65b77257fe9ef0f9d85ebc000f43143298c37bd4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2017 13:08:08 +0200 Subject: [PATCH 10/13] Add an explicit PyErr_Clear() and improve the comment. --- Python/symtable.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/symtable.c b/Python/symtable.c index 82c9943459ba37..bbac25cf3767aa 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1768,7 +1768,9 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, NULL, NULL) == -1) { if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { - /* Turn a DeprecationWarning into a SyntaxError. */ + /* Replace the DeprecationWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); PyErr_SetObject(PyExc_SyntaxError, msg); PyErr_SyntaxLocationObject(st->st_filename, st->st_cur->ste_lineno, From 978e863d33a1af94dcbb5626517af9c7cd2b55fb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2017 13:56:02 +0200 Subject: [PATCH 11/13] Fix minor doc errors. --- Doc/reference/expressions.rst | 8 +++++--- Doc/whatsnew/3.7.rst | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 9fdc5c14c61257..e776d00af2f6da 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -214,7 +214,8 @@ See also :pep:`530`. Asynchronous comprehensions were introduced. .. deprecated:: 3.7 - ``yield`` and ``yield from`` deprecated in the implicitly nested scope + ``yield`` and ``yield from`` deprecated in the implicitly nested scope. + .. _lists: @@ -365,7 +366,8 @@ which is an asynchronous iterator (see :ref:`async-iterators`). with 3.7, any function can use asynchronous generator expressions. .. deprecated:: 3.7 - ``yield`` and ``yield from`` deprecated in the implicitly nested scope + ``yield`` and ``yield from`` deprecated in the implicitly nested scope. + .. _yieldexpr: @@ -402,7 +404,7 @@ they will emit :exc:`SyntaxError`).. .. deprecated:: 3.7 Yield expressions deprecated in the implicitly nested scopes used to - implement comprehensions and generator expressions + implement comprehensions and generator expressions. Generator functions are described below, while asynchronous generator functions are described separately in section diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index fa2aa91006a8f9..4d232fe647bd8b 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -558,9 +558,9 @@ Deprecated * Yield expressions (both ``yield`` and ``yield from`` clauses) are now deprecated in comprehensions and generator expressions (aside from the outermost iterable - defined in the leftmost :keyword`for` clause). This ensures that comprehensions + defined in the leftmost :keyword:`for` clause). This ensures that comprehensions always immediately return a container of the appropriate type (rather than - potentially returning a :term:`generator-iterator` object), while generator + potentially returning a :term:`generator iterator` object), while generator expressions won't attempt to interleave their implicit output with the output from any explicit yield expressions. From 5287acf936f30de32d4033ee0f425024ba0f573c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 28 Nov 2017 07:34:53 +1000 Subject: [PATCH 12/13] Clarify eager evaluation of leftmost iterable - consistently use "leftmost" (never "outermost") - explicitly exclude the filter condition from eager evaluation - better align wording for comprehensions and generator expressions --- Doc/reference/expressions.rst | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index e776d00af2f6da..fb92ad0f07c21e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -183,14 +183,15 @@ by considering each of the :keyword:`for` or :keyword:`if` clauses a block, nesting from left to right, and evaluating the expression to produce an element each time the innermost block is reached. -However, aside from the leftmost :keyword:`for` clause, the comprehension is -executed in a separate implicitly nested scope. This ensures that names -assigned to in the target list don't "leak" into the enclosing scope. - -The leftmost :keyword:`for` clause is evaluated directly in the enclosing scope -and then passed as an argument to the implictly nested scope. Subsequent -:keyword:`for` clauses cannot be evaluated in the enclosing scope since they -may depend on the previous :keyword:`for` loop. For example: +However, aside from the iterable expression in the leftmost :keyword:`for` clause, +the comprehension is executed in a separate implicitly nested scope. This ensures +that names assigned to in the target list don't "leak" into the enclosing scope. + +The iterable expression in the leftmost :keyword:`for` clause is evaluated +directly in the enclosing scope and then passed as an argument to the implictly +nested scope. Subsequent :keyword:`for` clauses and any filter condition in the +leftmost :keyword:`for` clause cannot be evaluated in the enclosing scope as +they may depend on the values obtained from the leftmost iterable. For example: ``[x*y for x in range(10) for y in range(x, x+10)]``. To ensure the comprehension always results in a container of the appropriate @@ -335,12 +336,14 @@ brackets or curly braces. Variables used in the generator expression are evaluated lazily when the :meth:`~generator.__next__` method is called for the generator object (in the same -fashion as normal generators). However, the leftmost :keyword:`for` clause is -immediately evaluated, so that an error produced by it can be seen before any -other possible error in the code that handles the generator expression. -Subsequent :keyword:`for` clauses cannot be evaluated immediately since they -may depend on the previous :keyword:`for` loop. For example: ``(x*y for x in -range(10) for y in range(x, x+10))``. +fashion as normal generators). However, the iterable expression in the +leftmost :keyword:`for` clause is immediately evaluated, so that an error +produced by it will be emitted at the point where the generator expression +is defined, rather than at the point where the first value is retrieved. +Subsequent :keyword:`for` clauses and any filter condition in the leftmost +:keyword:`for` clause cannot be evaluated in the enclosing scope as they may +depend on the values obtained from the leftmost iterable. For example: +``(x*y for x in range(10) for y in range(x, x+10))``. The parentheses can be omitted on calls with only one argument. See section :ref:`calls` for details. From adb6268bc8ef7faa58d69b8277f5eea3c691c07d Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 28 Nov 2017 07:37:11 +1000 Subject: [PATCH 13/13] Tweak What's New wording --- Doc/whatsnew/3.7.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 4d232fe647bd8b..807e22b754f3d3 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -557,8 +557,8 @@ Deprecated ========== * Yield expressions (both ``yield`` and ``yield from`` clauses) are now deprecated - in comprehensions and generator expressions (aside from the outermost iterable - defined in the leftmost :keyword:`for` clause). This ensures that comprehensions + in comprehensions and generator expressions (aside from the iterable expression + in the leftmost :keyword:`for` clause). This ensures that comprehensions always immediately return a container of the appropriate type (rather than potentially returning a :term:`generator iterator` object), while generator expressions won't attempt to interleave their implicit output with the output