From 67c35204af609f9663ddf57b97d41e5f29bb009a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 13 Dec 2017 21:54:38 +0200 Subject: [PATCH 01/14] Add helpers --- Lib/asyncio/events.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index e425b06e42fa36..d7e8ff2e46349d 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -736,3 +736,23 @@ def set_child_watcher(watcher): """Equivalent to calling get_event_loop_policy().set_child_watcher(watcher).""" return get_event_loop_policy().set_child_watcher(watcher) + + +# shortcuts + +def create_task(coro): + """Schedule the execution of a coroutine object in a spawn task. + + Return a Task object. + """ + loop = get_running_loop() + return loop.create_task(coro) + + +def run_in_executor(executor, func, *args): + """Arrange for a func to be called in thread/process pool. + + This function is a coroutine: it returns a future instance. + """ + loop = get_running_loop() + return loop.run_in_executor(executor, func, *args) From d940cc3c2998f9543465bcdf0a88c297a65cc790 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 13 Dec 2017 22:30:26 +0200 Subject: [PATCH 02/14] Drop run_in_executor shortcut --- Lib/asyncio/events.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 778ecc6e618ce0..9a32dc1c4a859f 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -774,12 +774,3 @@ def create_task(coro): """ loop = get_running_loop() return loop.create_task(coro) - - -def run_in_executor(executor, func, *args): - """Arrange for a func to be called in thread/process pool. - - This function is a coroutine: it returns a future instance. - """ - loop = get_running_loop() - return loop.run_in_executor(executor, func, *args) From 298d38e5b7561d702af9c748813691a8a43154c2 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 13 Dec 2017 22:54:45 +0200 Subject: [PATCH 03/14] Move create_task() into tasks.py --- Lib/asyncio/events.py | 11 ----------- Lib/asyncio/tasks.py | 11 ++++++++++- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 9a32dc1c4a859f..a00f861a9e57b9 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -763,14 +763,3 @@ def set_child_watcher(watcher): _c__set_running_loop = _set_running_loop _c_get_running_loop = get_running_loop _c_get_event_loop = get_event_loop - - -# shortcuts - -def create_task(coro): - """Schedule the execution of a coroutine object in a spawn task. - - Return a Task object. - """ - loop = get_running_loop() - return loop.create_task(coro) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index c5122f760715ac..35350843309b4d 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -1,7 +1,7 @@ """Support for tasks, coroutines and the scheduler.""" __all__ = ( - 'Task', + 'Task', 'create_task', 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', 'wait', 'wait_for', 'as_completed', 'sleep', 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', @@ -263,6 +263,15 @@ def _wakeup(self, future): Task = _CTask = _asyncio.Task +def create_task(coro): + """Schedule the execution of a coroutine object in a spawn task. + + Return a Task object. + """ + loop = get_running_loop() + return loop.create_task(coro) + + # wait() and as_completed() similar to those in PEP 3148. FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED From 8ca254e08a7890557a3fad4e9ec515f12248d60c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 11:51:52 +0200 Subject: [PATCH 04/14] Extract asyncio.utils, add runtime checks to task ctors --- Lib/asyncio/__init__.py | 4 +- Lib/asyncio/base_futures.py | 4 +- Lib/asyncio/constants.py | 2 +- Lib/asyncio/coroutines.py | 10 ++--- Lib/asyncio/events.py | 81 +++------------------------------- Lib/asyncio/futures.py | 3 +- Lib/asyncio/tasks.py | 3 +- Lib/asyncio/utils.py | 78 ++++++++++++++++++++++++++++++++ Lib/test/test_asyncio/utils.py | 3 +- Modules/_asynciomodule.c | 23 ++++++++++ 10 files changed, 123 insertions(+), 88 deletions(-) create mode 100644 Lib/asyncio/utils.py diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index dd6686de84056a..cda091235f21fa 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -16,6 +16,7 @@ from .subprocess import * from .tasks import * from .transports import * +from .utils import * __all__ = (base_events.__all__ + coroutines.__all__ + @@ -27,7 +28,8 @@ streams.__all__ + subprocess.__all__ + tasks.__all__ + - transports.__all__) + transports.__all__ + + utils.__all__) if sys.platform == 'win32': # pragma: no cover from .windows_events import * diff --git a/Lib/asyncio/base_futures.py b/Lib/asyncio/base_futures.py index 2ee82c3f057033..9b3789eb230af0 100644 --- a/Lib/asyncio/base_futures.py +++ b/Lib/asyncio/base_futures.py @@ -3,7 +3,7 @@ import concurrent.futures._base import reprlib -from . import events +from . import utils Error = concurrent.futures._base.Error CancelledError = concurrent.futures.CancelledError @@ -38,7 +38,7 @@ def _format_callbacks(cb): cb = '' def format_cb(callback): - return events._format_callback_source(callback, ()) + return utils._format_callback_source(callback, ()) if size == 1: cb = format_cb(cb[0]) diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index dfe97f4985969f..823cef70695150 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -6,5 +6,5 @@ # Number of stack entries to capture in debug mode. # The larger the number, the slower the operation in debug mode -# (see extract_stack() in events.py). +# (see extract_stack() in utils.py). DEBUG_STACK_DEPTH = 10 diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index bca7fe3a53742d..6d86406bd3c192 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -9,9 +9,9 @@ from collections.abc import Awaitable, Coroutine -from . import constants -from . import events from . import base_futures +from . import constants +from . import utils from .log import logger @@ -48,7 +48,7 @@ def __init__(self, gen, func=None): assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen self.gen = gen self.func = func # Used to unwrap @coroutine decorator - self._source_traceback = events.extract_stack(sys._getframe(1)) + self._source_traceback = utils.extract_stack(sys._getframe(1)) self.__name__ = getattr(gen, '__name__', None) self.__qualname__ = getattr(gen, '__qualname__', None) @@ -243,7 +243,7 @@ def _format_coroutine(coro): func = coro if coro_name is None: - coro_name = events._format_callback(func, (), {}) + coro_name = utils._format_callback(func, (), {}) try: coro_code = coro.gi_code @@ -260,7 +260,7 @@ def _format_coroutine(coro): if (isinstance(coro, CoroWrapper) and not inspect.isgeneratorfunction(coro.func) and coro.func is not None): - source = events._get_function_source(coro.func) + source = utils._get_function_source(coro.func) if source is not None: filename, lineno = source if coro_frame is None: diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index a00f861a9e57b9..b43eff2bcd9a81 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -11,86 +11,14 @@ '_get_running_loop', ) -import functools -import inspect import os -import reprlib import socket import subprocess import sys import threading -import traceback from . import constants - - -def _get_function_source(func): - func = inspect.unwrap(func) - if inspect.isfunction(func): - code = func.__code__ - return (code.co_filename, code.co_firstlineno) - if isinstance(func, functools.partial): - return _get_function_source(func.func) - if isinstance(func, functools.partialmethod): - return _get_function_source(func.func) - return None - - -def _format_args_and_kwargs(args, kwargs): - """Format function arguments and keyword arguments. - - Special case for a single parameter: ('hello',) is formatted as ('hello'). - """ - # use reprlib to limit the length of the output - items = [] - if args: - items.extend(reprlib.repr(arg) for arg in args) - if kwargs: - items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items()) - return '({})'.format(', '.join(items)) - - -def _format_callback(func, args, kwargs, suffix=''): - if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) - - if hasattr(func, '__qualname__'): - func_repr = getattr(func, '__qualname__') - elif hasattr(func, '__name__'): - func_repr = getattr(func, '__name__') - else: - func_repr = repr(func) - - func_repr += _format_args_and_kwargs(args, kwargs) - if suffix: - func_repr += suffix - return func_repr - - -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) - source = _get_function_source(func) - if source: - func_repr += f' at {source[0]}:{source[1]}' - return func_repr - - -def extract_stack(f=None, limit=None): - """Replacement for traceback.extract_stack() that only does the - necessary work for asyncio debug mode. - """ - if f is None: - f = sys._getframe().f_back - if limit is None: - # Limit the amount of work to a reasonable amount, as extract_stack() - # can be called for each coroutine and future in debug mode. - limit = constants.DEBUG_STACK_DEPTH - stack = traceback.StackSummary.extract(traceback.walk_stack(f), - limit=limit, - lookup_lines=False) - stack.reverse() - return stack +from . import utils class Handle: @@ -106,7 +34,7 @@ def __init__(self, callback, args, loop): self._cancelled = False self._repr = None if self._loop.get_debug(): - self._source_traceback = extract_stack(sys._getframe(1)) + self._source_traceback = utils.extract_stack(sys._getframe(1)) else: self._source_traceback = None @@ -115,7 +43,8 @@ def _repr_info(self): if self._cancelled: info.append('cancelled') if self._callback is not None: - info.append(_format_callback_source(self._callback, self._args)) + info.append(utils._format_callback_source(self._callback, + self._args)) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -145,7 +74,7 @@ def _run(self): try: self._callback(*self._args) except Exception as exc: - cb = _format_callback_source(self._callback, self._args) + cb = utils._format_callback_source(self._callback, self._args) msg = f'Exception in callback {cb}' context = { 'message': msg, diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index d46a295197b876..2523c60e85755b 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -11,6 +11,7 @@ from . import base_futures from . import events +from . import utils CancelledError = base_futures.CancelledError @@ -79,7 +80,7 @@ def __init__(self, *, loop=None): self._loop = loop self._callbacks = [] if self._loop.get_debug(): - self._source_traceback = events.extract_stack(sys._getframe(1)) + self._source_traceback = utils.extract_stack(sys._getframe(1)) _repr_info = base_futures._future_repr_info diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 35350843309b4d..5ea564b05999ac 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -67,7 +67,8 @@ def all_tasks(cls, loop=None): return {t for t in cls._all_tasks if t._loop is loop} def __init__(self, coro, *, loop=None): - assert coroutines.iscoroutine(coro), repr(coro) + if not coroutines.iscoroutine(coro): + raise TypeError(f"{coro!r} is not a coroutine") super().__init__(loop=loop) if self._source_traceback: del self._source_traceback[-1] diff --git a/Lib/asyncio/utils.py b/Lib/asyncio/utils.py new file mode 100644 index 00000000000000..78447c51f68d24 --- /dev/null +++ b/Lib/asyncio/utils.py @@ -0,0 +1,78 @@ +import functools +import inspect +import reprlib +import traceback + +from . import constants + + +__all__ = ('extract_stack',) + + +def _get_function_source(func): + func = inspect.unwrap(func) + if inspect.isfunction(func): + code = func.__code__ + return (code.co_filename, code.co_firstlineno) + if isinstance(func, functools.partial): + return _get_function_source(func.func) + if isinstance(func, functools.partialmethod): + return _get_function_source(func.func) + return None + + +def _format_callback_source(func, args): + func_repr = _format_callback(func, args, None) + source = _get_function_source(func) + if source: + func_repr += f' at {source[0]}:{source[1]}' + return func_repr + + +def _format_args_and_kwargs(args, kwargs): + """Format function arguments and keyword arguments. + + Special case for a single parameter: ('hello',) is formatted as ('hello'). + """ + # use reprlib to limit the length of the output + items = [] + if args: + items.extend(reprlib.repr(arg) for arg in args) + if kwargs: + items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items()) + return '({})'.format(', '.join(items)) + + +def _format_callback(func, args, kwargs, suffix=''): + if isinstance(func, functools.partial): + suffix = _format_args_and_kwargs(args, kwargs) + suffix + return _format_callback(func.func, func.args, func.keywords, suffix) + + if hasattr(func, '__qualname__'): + func_repr = getattr(func, '__qualname__') + elif hasattr(func, '__name__'): + func_repr = getattr(func, '__name__') + else: + func_repr = repr(func) + + func_repr += _format_args_and_kwargs(args, kwargs) + if suffix: + func_repr += suffix + return func_repr + + +def extract_stack(f=None, limit=None): + """Replacement for traceback.extract_stack() that only does the + necessary work for asyncio debug mode. + """ + if f is None: + f = sys._getframe().f_back + if limit is None: + # Limit the amount of work to a reasonable amount, as extract_stack() + # can be called for each coroutine and future in debug mode. + limit = constants.DEBUG_STACK_DEPTH + stack = traceback.StackSummary.extract(traceback.walk_stack(f), + limit=limit, + lookup_lines=False) + stack.reverse() + return stack diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 560db9f562d82f..0e1adcc5fba8ba 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -30,6 +30,7 @@ from asyncio import events from asyncio import futures from asyncio import tasks +from asyncio import utils from asyncio.log import logger from test import support @@ -429,7 +430,7 @@ def __eq__(self, other): def get_function_source(func): - source = events._get_function_source(func) + source = utils._get_function_source(func) if source is None: raise ValueError("unable to get the source of %r" % (func,)) return source diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 01c38b80b95f8f..6bc58e873fa503 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -26,6 +26,7 @@ static PyObject *all_tasks; static PyObject *current_tasks; static PyObject *traceback_extract_stack; static PyObject *asyncio_get_event_loop_policy; +static PyObject *asyncio_iscoroutine_func; static PyObject *asyncio_future_repr_info_func; static PyObject *asyncio_task_repr_info_func; static PyObject *asyncio_task_get_stack_func; @@ -1461,8 +1462,26 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop) /*[clinic end generated code: output=9f24774c2287fc2f input=8d132974b049593e]*/ { PyObject *res; + int tmp; _Py_IDENTIFIER(add); + if (!PyCoro_CheckExact(coro)) { + // fastpath failed, perfom slow check + res = PyObject_CallFunctionObjArgs(asyncio_iscoroutine_func, + coro, NULL); + if (res == NULL) { + return -1; + } + tmp = PyObject_Not(res); + Py_DECREF(res); + if (tmp < 0) { + return -1; + } + if (tmp) { + PyErr_Format(PyExc_TypeError, "%R is not a coroutine", coro, NULL); + return -1; + } + } if (future_init((FutureObj*)self, loop)) { return -1; } @@ -2604,6 +2623,7 @@ module_free(void *m) Py_CLEAR(traceback_extract_stack); Py_CLEAR(asyncio_get_event_loop_policy); Py_CLEAR(asyncio_future_repr_info_func); + Py_CLEAR(asyncio_iscoroutine_func); Py_CLEAR(asyncio_task_repr_info_func); Py_CLEAR(asyncio_task_get_stack_func); Py_CLEAR(asyncio_task_print_stack_func); @@ -2645,6 +2665,9 @@ module_init(void) GET_MOD_ATTR(asyncio_task_get_stack_func, "_task_get_stack") GET_MOD_ATTR(asyncio_task_print_stack_func, "_task_print_stack") + WITH_MOD("asyncio.coroutines") + GET_MOD_ATTR(asyncio_iscoroutine_func, "iscoroutine") + WITH_MOD("inspect") GET_MOD_ATTR(inspect_isgenerator, "isgenerator") From a25f1cba0c7af0d7f913cbce6bac4257b35fbb1a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 12:21:45 +0200 Subject: [PATCH 05/14] Add tests --- Lib/asyncio/tasks.py | 13 +++++++++---- Lib/test/test_asyncio/test_tasks.py | 23 +++++++++++++++++++++++ Modules/_asynciomodule.c | 10 ++++++---- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 5ea564b05999ac..35cfae2500e483 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -67,14 +67,19 @@ def all_tasks(cls, loop=None): return {t for t in cls._all_tasks if t._loop is loop} def __init__(self, coro, *, loop=None): - if not coroutines.iscoroutine(coro): - raise TypeError(f"{coro!r} is not a coroutine") super().__init__(loop=loop) if self._source_traceback: del self._source_traceback[-1] - self._coro = coro - self._fut_waiter = None + if not coroutines.iscoroutine(coro): + # raise after Future.__init__(), attrs are required for __del__ + # prevent logging for pending task in __del__ + self._log_destroy_pending = False + raise TypeError(f"{coro!r} is not a coroutine") + self._must_cancel = False + self._fut_waiter = None + self._coro = coro + self._loop.call_soon(self._step) self.__class__._all_tasks.add(self) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 0838ebf3a7b4f7..6b529860475aea 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2054,6 +2054,29 @@ def coro(): self.assertEqual(self.Task.all_tasks(self.loop), set()) + def test_create_task_with_noncoroutine(self): + with self.assertRaises(TypeError): + self.new_task(self.loop, 123) + + def test_create_task_with_oldstyle_coroutine(self): + + @asyncio.coroutine + def coro(): + pass + + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + def test_create_task_with_async_function(self): + + async def coro(): + pass + + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + def add_subclass_tests(cls): BaseTask = cls.Task diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 6bc58e873fa503..fd36af8dff5ca8 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1465,8 +1465,13 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop) int tmp; _Py_IDENTIFIER(add); + if (future_init((FutureObj*)self, loop)) { + return -1; + } + if (!PyCoro_CheckExact(coro)) { // fastpath failed, perfom slow check + // raise after Future.__init__(), attrs are required for __del__ res = PyObject_CallFunctionObjArgs(asyncio_iscoroutine_func, coro, NULL); if (res == NULL) { @@ -1478,18 +1483,15 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop) return -1; } if (tmp) { + self->task_log_destroy_pending = 0; PyErr_Format(PyExc_TypeError, "%R is not a coroutine", coro, NULL); return -1; } } - if (future_init((FutureObj*)self, loop)) { - return -1; - } self->task_fut_waiter = NULL; self->task_must_cancel = 0; self->task_log_destroy_pending = 1; - Py_INCREF(coro); self->task_coro = coro; From ef9ccda913d382ce259cb4a812705ed6d409e879 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 12:27:15 +0200 Subject: [PATCH 06/14] utils.py -> format_helpers.py --- Lib/asyncio/__init__.py | 6 +++--- Lib/asyncio/base_futures.py | 4 ++-- Lib/asyncio/coroutines.py | 8 ++++---- Lib/asyncio/events.py | 12 +++++++----- Lib/asyncio/{utils.py => format_helpers.py} | 0 Lib/asyncio/futures.py | 5 +++-- Lib/test/test_asyncio/utils.py | 4 ++-- 7 files changed, 21 insertions(+), 18 deletions(-) rename Lib/asyncio/{utils.py => format_helpers.py} (100%) diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index cda091235f21fa..4510e0aa9cfd30 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -8,6 +8,7 @@ from .base_events import * from .coroutines import * from .events import * +from .format_helpers import * from .futures import * from .locks import * from .protocols import * @@ -16,11 +17,11 @@ from .subprocess import * from .tasks import * from .transports import * -from .utils import * __all__ = (base_events.__all__ + coroutines.__all__ + events.__all__ + + format_helpers.__all__ + futures.__all__ + locks.__all__ + protocols.__all__ + @@ -28,8 +29,7 @@ streams.__all__ + subprocess.__all__ + tasks.__all__ + - transports.__all__ + - utils.__all__) + transports.__all__) if sys.platform == 'win32': # pragma: no cover from .windows_events import * diff --git a/Lib/asyncio/base_futures.py b/Lib/asyncio/base_futures.py index 9b3789eb230af0..008812eda91bbf 100644 --- a/Lib/asyncio/base_futures.py +++ b/Lib/asyncio/base_futures.py @@ -3,7 +3,7 @@ import concurrent.futures._base import reprlib -from . import utils +from . import format_helpers Error = concurrent.futures._base.Error CancelledError = concurrent.futures.CancelledError @@ -38,7 +38,7 @@ def _format_callbacks(cb): cb = '' def format_cb(callback): - return utils._format_callback_source(callback, ()) + return format_helpers._format_callback_source(callback, ()) if size == 1: cb = format_cb(cb[0]) diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index 6d86406bd3c192..e3c0162dd1458c 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -11,7 +11,7 @@ from . import base_futures from . import constants -from . import utils +from . import format_helpers from .log import logger @@ -48,7 +48,7 @@ def __init__(self, gen, func=None): assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen self.gen = gen self.func = func # Used to unwrap @coroutine decorator - self._source_traceback = utils.extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack(sys._getframe(1)) self.__name__ = getattr(gen, '__name__', None) self.__qualname__ = getattr(gen, '__qualname__', None) @@ -243,7 +243,7 @@ def _format_coroutine(coro): func = coro if coro_name is None: - coro_name = utils._format_callback(func, (), {}) + coro_name = format_helpers._format_callback(func, (), {}) try: coro_code = coro.gi_code @@ -260,7 +260,7 @@ def _format_coroutine(coro): if (isinstance(coro, CoroWrapper) and not inspect.isgeneratorfunction(coro.func) and coro.func is not None): - source = utils._get_function_source(coro.func) + source = format_helpers._get_function_source(coro.func) if source is not None: filename, lineno = source if coro_frame is None: diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index b43eff2bcd9a81..974a4a22218fd5 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -18,7 +18,7 @@ import threading from . import constants -from . import utils +from . import format_helpers class Handle: @@ -34,7 +34,8 @@ def __init__(self, callback, args, loop): self._cancelled = False self._repr = None if self._loop.get_debug(): - self._source_traceback = utils.extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) else: self._source_traceback = None @@ -43,8 +44,8 @@ def _repr_info(self): if self._cancelled: info.append('cancelled') if self._callback is not None: - info.append(utils._format_callback_source(self._callback, - self._args)) + info.append(format_helpers._format_callback_source( + self._callback, self._args)) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -74,7 +75,8 @@ def _run(self): try: self._callback(*self._args) except Exception as exc: - cb = utils._format_callback_source(self._callback, self._args) + cb = format_helpers._format_callback_source( + self._callback, self._args) msg = f'Exception in callback {cb}' context = { 'message': msg, diff --git a/Lib/asyncio/utils.py b/Lib/asyncio/format_helpers.py similarity index 100% rename from Lib/asyncio/utils.py rename to Lib/asyncio/format_helpers.py diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 2523c60e85755b..b310962f9fe19d 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -11,7 +11,7 @@ from . import base_futures from . import events -from . import utils +from . import format_helpers CancelledError = base_futures.CancelledError @@ -80,7 +80,8 @@ def __init__(self, *, loop=None): self._loop = loop self._callbacks = [] if self._loop.get_debug(): - self._source_traceback = utils.extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) _repr_info = base_futures._future_repr_info diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 0e1adcc5fba8ba..a1a9bb3684c3a9 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -28,9 +28,9 @@ from asyncio import base_events from asyncio import events +from asyncio import format_helpers from asyncio import futures from asyncio import tasks -from asyncio import utils from asyncio.log import logger from test import support @@ -430,7 +430,7 @@ def __eq__(self, other): def get_function_source(func): - source = utils._get_function_source(func) + source = format_helpers._get_function_source(func) if source is None: raise ValueError("unable to get the source of %r" % (func,)) return source From 7cbb758fac7078575529f31211936a77b4e5dad5 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 17:20:21 +0200 Subject: [PATCH 07/14] Don't expose asyncio.extract_stack --- Lib/asyncio/__init__.py | 2 -- Lib/asyncio/format_helpers.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 4510e0aa9cfd30..dd6686de84056a 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -8,7 +8,6 @@ from .base_events import * from .coroutines import * from .events import * -from .format_helpers import * from .futures import * from .locks import * from .protocols import * @@ -21,7 +20,6 @@ __all__ = (base_events.__all__ + coroutines.__all__ + events.__all__ + - format_helpers.__all__ + futures.__all__ + locks.__all__ + protocols.__all__ + diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py index 78447c51f68d24..39cfcee0c1c8bf 100644 --- a/Lib/asyncio/format_helpers.py +++ b/Lib/asyncio/format_helpers.py @@ -6,9 +6,6 @@ from . import constants -__all__ = ('extract_stack',) - - def _get_function_source(func): func = inspect.unwrap(func) if inspect.isfunction(func): From 14b2588ac08586644d0a51920792ad26ed60fa4d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 17:25:17 +0200 Subject: [PATCH 08/14] Add test for asyncio.create_task() --- Lib/test/test_asyncio/test_tasks.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 6b529860475aea..9bd83d6127f488 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2077,6 +2077,19 @@ async def coro(): self.assertIsInstance(task, self.Task) self.loop.run_until_complete(task) + def test_bare_create_task(self): + + async def inner(): + return 1 + + async def coro(): + task = asyncio.create_task(inner()) + self.assertIsInstance(task, self.Task) + ret = await task + self.assertEqual(1, ret) + + self.loop.run_until_complete(coro()) + def add_subclass_tests(cls): BaseTask = cls.Task From e1c1ed1301ec0e43a44248eb647f49a1413dfcee Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 17:26:49 +0200 Subject: [PATCH 09/14] Fix implementation --- Lib/asyncio/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 35cfae2500e483..4fe78e9e0627cf 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -274,7 +274,7 @@ def create_task(coro): Return a Task object. """ - loop = get_running_loop() + loop = events.get_running_loop() return loop.create_task(coro) From 8533c7404c295efad64c0b80b1c2d5588ae16f30 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 17:29:07 +0200 Subject: [PATCH 10/14] Add NEWS entry --- .../NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst diff --git a/Misc/NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst b/Misc/NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst new file mode 100644 index 00000000000000..e2d10959476cdc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst @@ -0,0 +1 @@ +Implement asyncio.create_task(coro) shortcut From 0f6ca60c2fb58d3b42a12a0e4493492ea3e49aa3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 17:47:20 +0200 Subject: [PATCH 11/14] Update docs --- Doc/library/asyncio-task.rst | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index a8a0a8e85efe72..749b4abb51ba4f 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -358,10 +358,21 @@ with the result. Task ---- +.. function:: create_task(coro) + + Schedule the execution of a :ref:`coroutine ` object + *coro* in a spawn task. + + The task is executed in :func:`get_running_loop` context, + :exc:`RuntimeError` is raised if there is no running loop in + current thread. + + .. versionadded:: 3.7 + .. class:: Task(coro, \*, loop=None) - Schedule the execution of a :ref:`coroutine `: wrap it in a - future. A task is a subclass of :class:`Future`. + A unit for concurrent running of :ref:`coroutines `, + subclass of :class:`Future`. A task is responsible for executing a coroutine object in an event loop. If the wrapped coroutine yields from a future, the task suspends the execution @@ -386,7 +397,7 @@ Task ` did not complete. It is probably a bug and a warning is logged: see :ref:`Pending task destroyed `. - Don't directly create :class:`Task` instances: use the :func:`ensure_future` + Don't directly create :class:`Task` instances: use the :func:`create_task` function or the :meth:`AbstractEventLoop.create_task` method. This class is :ref:`not thread safe `. @@ -534,9 +545,15 @@ Task functions .. versionchanged:: 3.5.1 The function accepts any :term:`awaitable` object. + .. note:: + + :func:`create_task` (added in Python 3.7) is the preferable way + for spawning new tasks. + .. seealso:: - The :meth:`AbstractEventLoop.create_task` method. + The :func:`create_task` function and + :meth:`AbstractEventLoop.create_task` method. .. function:: wrap_future(future, \*, loop=None) From a5d4a8bc05f8da7e25333c5b0b9f89b8abd7e163 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 18:18:33 +0200 Subject: [PATCH 12/14] Change exception message --- Lib/asyncio/tasks.py | 2 +- Lib/test/test_asyncio/test_tasks.py | 3 ++- Modules/_asynciomodule.c | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 4fe78e9e0627cf..172057e5a29a53 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -74,7 +74,7 @@ def __init__(self, coro, *, loop=None): # raise after Future.__init__(), attrs are required for __del__ # prevent logging for pending task in __del__ self._log_destroy_pending = False - raise TypeError(f"{coro!r} is not a coroutine") + raise TypeError(f"a coroutine was expected, got {coro!r}") self._must_cancel = False self._fut_waiter = None diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 9bd83d6127f488..6e2a16f2a0556b 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2055,7 +2055,8 @@ def coro(): self.assertEqual(self.Task.all_tasks(self.loop), set()) def test_create_task_with_noncoroutine(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, + "a coroutine was expected, got 123"): self.new_task(self.loop, 123) def test_create_task_with_oldstyle_coroutine(self): diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index fd36af8dff5ca8..9ac1c44d48d380 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1484,7 +1484,9 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop) } if (tmp) { self->task_log_destroy_pending = 0; - PyErr_Format(PyExc_TypeError, "%R is not a coroutine", coro, NULL); + PyErr_Format(PyExc_TypeError, + "a coroutine was expected, got %R", + coro, NULL); return -1; } } From 06662766b87b724cfa23206d6c7189598a1264b9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 18:53:09 +0200 Subject: [PATCH 13/14] Update doc --- Doc/library/asyncio-task.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 749b4abb51ba4f..243bfd7873603b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -360,8 +360,8 @@ Task .. function:: create_task(coro) - Schedule the execution of a :ref:`coroutine ` object - *coro* in a spawn task. + Wrap a :ref:`coroutine ` *coro* into a task and schedule + its execution. The task is executed in :func:`get_running_loop` context, :exc:`RuntimeError` is raised if there is no running loop in From 5dbae4e32c3eafe3ff1a591a395ea5d9fbbaea09 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 14 Dec 2017 23:05:57 +0200 Subject: [PATCH 14/14] Fix notes --- Doc/library/asyncio-task.rst | 2 +- Lib/asyncio/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 243bfd7873603b..72458c8cfd62e3 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -361,7 +361,7 @@ Task .. function:: create_task(coro) Wrap a :ref:`coroutine ` *coro* into a task and schedule - its execution. + its execution. Return the task object. The task is executed in :func:`get_running_loop` context, :exc:`RuntimeError` is raised if there is no running loop in diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index 823cef70695150..52169c3f8e5b12 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -6,5 +6,5 @@ # Number of stack entries to capture in debug mode. # The larger the number, the slower the operation in debug mode -# (see extract_stack() in utils.py). +# (see extract_stack() in format_helpers.py). DEBUG_STACK_DEPTH = 10