From f7fe1c6297c59b1f8b6d7ac9060acb56db0f33d6 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Mon, 26 Jun 2017 15:01:31 +0200 Subject: [PATCH 1/3] bpo-30695: Add set_nomemory(start, stop) to _testcapi --- Lib/test/test_capi.py | 34 +++++++- Modules/_testcapimodule.c | 170 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 1cf5cd73f09391..c3a04b49df6e8c 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -15,7 +15,7 @@ import unittest from test import support from test.support import MISSING_C_DOCSTRINGS -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok try: import _posixsubprocess except ImportError: @@ -243,6 +243,38 @@ def test_return_result_with_error(self): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() + def test_set_nomemory(self): + code = """if 1: + import _testcapi + + class C(): pass + + # The first loop tests both functions and that remove_mem_hooks() + # can be called twice in a row. The second loop checks a call to + # set_nomemory() after a call to remove_mem_hooks(). The third + # loop checks the start and stop arguments of set_nomemory(). + for outer_cnt in range(1, 4): + start = 10 * outer_cnt + for j in range(100): + if j == 0: + if outer_cnt != 3: + _testcapi.set_nomemory(start) + else: + _testcapi.set_nomemory(start, start + 1) + try: + C() + except MemoryError as e: + if outer_cnt != 3: + _testcapi.remove_mem_hooks() + print('MemoryError', outer_cnt, j) + _testcapi.remove_mem_hooks() + break + """ + rc, out, err = assert_python_ok('-c', code) + self.assertIn(b'MemoryError 1 10', out) + self.assertIn(b'MemoryError 2 20', out) + self.assertIn(b'MemoryError 3 30', out) + @unittest.skipUnless(threading, 'Threading required for this test.') class TestPendingCalls(unittest.TestCase): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 83ba33dd7acfec..b86808f286a2ae 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3420,6 +3420,172 @@ test_pyobject_setallocators(PyObject *self) return test_setallocators(PYMEM_DOMAIN_OBJ); } +/* Part of the following code is inherited from the pyfailmalloc project + * written by Victor Stinner. */ +static struct { + int installed; + PyMemAllocatorEx raw; + PyMemAllocatorEx mem; + PyMemAllocatorEx obj; +} FmHook; + +typedef struct fm_filter_t { + void *data; + int (*has_memory) (struct fm_filter_t *); +} fm_filter_t; + +static fm_filter_t *FmFilter; + +static void * +hook_fmalloc(void *ctx, size_t size) +{ + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + if (FmFilter != NULL && ! FmFilter->has_memory(FmFilter)) { + return NULL; + } + return alloc->malloc(alloc->ctx, size); +} + +static void * +hook_fcalloc(void *ctx, size_t nelem, size_t elsize) +{ + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + if (FmFilter != NULL && ! FmFilter->has_memory(FmFilter)) { + return NULL; + } + return alloc->calloc(alloc->ctx, nelem, elsize); +} + +static void * +hook_frealloc(void *ctx, void *ptr, size_t new_size) +{ + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + if (FmFilter != NULL && ! FmFilter->has_memory(FmFilter)) { + return NULL; + } + return alloc->realloc(alloc->ctx, ptr, new_size); +} + +static void +hook_ffree(void *ctx, void *ptr) +{ + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + alloc->free(alloc->ctx, ptr); +} + +static void +fm_setup_hooks(void) +{ + PyMemAllocatorEx alloc; + + if (FmHook.installed) { + return; + } + FmHook.installed = 1; + + alloc.malloc = hook_fmalloc; + alloc.calloc = hook_fcalloc; + alloc.realloc = hook_frealloc; + alloc.free = hook_ffree; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &FmHook.raw); + PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &FmHook.mem); + PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &FmHook.obj); + + alloc.ctx = &FmHook.raw; + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); + + alloc.ctx = &FmHook.mem; + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + + alloc.ctx = &FmHook.obj; + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); +} + +static void +fm_remove_hooks(void) +{ + if (FmHook.installed) { + FmHook.installed = 0; + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &FmHook.raw); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &FmHook.mem); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &FmHook.obj); + } +} + +static void +fm_remove_filter(void) +{ + if (FmFilter != NULL) { + if (FmFilter->data != NULL) { + PyMem_RawFree(FmFilter->data); + } + PyMem_RawFree(FmFilter); + FmFilter = NULL; + } +} + +typedef struct { + int count; + int start; + int stop; +} nomem_data_t; + +static int +nomem_func(fm_filter_t *filter) +{ + nomem_data_t *data = (nomem_data_t *)filter->data; + /* Return true if memory should be made available. */ + data->count++; + if (data->count > data->start && + (data->stop <= 0 || data->count <= data->stop)) { + return 0; + } + return 1; +} + +static PyObject* +set_nomemory(PyObject *self, PyObject *args) +{ + nomem_data_t *data; + + fm_remove_filter(); + data = (nomem_data_t *)PyMem_RawMalloc(sizeof(nomem_data_t)); + if (data == NULL) { + goto error; + } + FmFilter = (fm_filter_t *)PyMem_RawMalloc(sizeof(fm_filter_t)); + if (FmFilter == NULL) { + PyMem_RawFree(data); + goto error; + } + FmFilter->data = data; + FmFilter->has_memory = nomem_func; + + /* Memory allocation fails after 'start' allocation requests, and until + * 'stop' allocation requests except when 'stop' is negative or equal + * to 0 (default) in which case allocation failures never stop. */ + data->count = 0; + data->stop = 0; + if (! PyArg_ParseTuple(args, "i|i", &data->start, &data->stop)) { + fm_remove_filter(); + return NULL; + } + fm_setup_hooks(); + Py_RETURN_NONE; + +error: + PyErr_SetString(PyExc_RuntimeError, "PyMem_RawMalloc() returns NULL"); + return NULL; +} + +static PyObject* +remove_mem_hooks(PyObject *self) +{ + fm_remove_hooks(); + fm_remove_filter(); + Py_RETURN_NONE; +} + PyDoc_STRVAR(docstring_empty, "" ); @@ -4287,6 +4453,10 @@ static PyMethodDef TestMethods[] = { (PyCFunction)test_pymem_setallocators, METH_NOARGS}, {"test_pyobject_setallocators", (PyCFunction)test_pyobject_setallocators, METH_NOARGS}, + {"set_nomemory", (PyCFunction)set_nomemory, METH_VARARGS, + PyDoc_STR("set_nomemory(start:int, stop:int = 0)")}, + {"remove_mem_hooks", (PyCFunction)remove_mem_hooks, METH_NOARGS, + PyDoc_STR("Remove memory hooks.")}, {"no_docstring", (PyCFunction)test_with_docstring, METH_NOARGS}, {"docstring_empty", From 3b46e972e39ca594a0ef52e4e97ce4118674f0b3 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Wed, 28 Jun 2017 12:33:02 +0200 Subject: [PATCH 2/3] remove fm_filter_t to simplify the implementation --- Modules/_testcapimodule.c | 86 ++++++++++----------------------------- 1 file changed, 22 insertions(+), 64 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b86808f286a2ae..c9c279864369b3 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3420,7 +3420,7 @@ test_pyobject_setallocators(PyObject *self) return test_setallocators(PYMEM_DOMAIN_OBJ); } -/* Part of the following code is inherited from the pyfailmalloc project +/* Most part of the following code is inherited from the pyfailmalloc project * written by Victor Stinner. */ static struct { int installed; @@ -3429,18 +3429,28 @@ static struct { PyMemAllocatorEx obj; } FmHook; -typedef struct fm_filter_t { - void *data; - int (*has_memory) (struct fm_filter_t *); -} fm_filter_t; +static struct { + int start; + int stop; + Py_ssize_t count; +} FmData; -static fm_filter_t *FmFilter; +static int +fm_nomemory(void) +{ + FmData.count++; + if (FmData.count > FmData.start && + (FmData.stop <= 0 || FmData.count <= FmData.stop)) { + return 1; + } + return 0; +} static void * hook_fmalloc(void *ctx, size_t size) { PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - if (FmFilter != NULL && ! FmFilter->has_memory(FmFilter)) { + if (fm_nomemory()) { return NULL; } return alloc->malloc(alloc->ctx, size); @@ -3450,7 +3460,7 @@ static void * hook_fcalloc(void *ctx, size_t nelem, size_t elsize) { PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - if (FmFilter != NULL && ! FmFilter->has_memory(FmFilter)) { + if (fm_nomemory()) { return NULL; } return alloc->calloc(alloc->ctx, nelem, elsize); @@ -3460,7 +3470,7 @@ static void * hook_frealloc(void *ctx, void *ptr, size_t new_size) { PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - if (FmFilter != NULL && ! FmFilter->has_memory(FmFilter)) { + if (fm_nomemory()) { return NULL; } return alloc->realloc(alloc->ctx, ptr, new_size); @@ -3512,77 +3522,25 @@ fm_remove_hooks(void) } } -static void -fm_remove_filter(void) -{ - if (FmFilter != NULL) { - if (FmFilter->data != NULL) { - PyMem_RawFree(FmFilter->data); - } - PyMem_RawFree(FmFilter); - FmFilter = NULL; - } -} - -typedef struct { - int count; - int start; - int stop; -} nomem_data_t; - -static int -nomem_func(fm_filter_t *filter) -{ - nomem_data_t *data = (nomem_data_t *)filter->data; - /* Return true if memory should be made available. */ - data->count++; - if (data->count > data->start && - (data->stop <= 0 || data->count <= data->stop)) { - return 0; - } - return 1; -} - static PyObject* set_nomemory(PyObject *self, PyObject *args) { - nomem_data_t *data; - - fm_remove_filter(); - data = (nomem_data_t *)PyMem_RawMalloc(sizeof(nomem_data_t)); - if (data == NULL) { - goto error; - } - FmFilter = (fm_filter_t *)PyMem_RawMalloc(sizeof(fm_filter_t)); - if (FmFilter == NULL) { - PyMem_RawFree(data); - goto error; - } - FmFilter->data = data; - FmFilter->has_memory = nomem_func; - /* Memory allocation fails after 'start' allocation requests, and until * 'stop' allocation requests except when 'stop' is negative or equal * to 0 (default) in which case allocation failures never stop. */ - data->count = 0; - data->stop = 0; - if (! PyArg_ParseTuple(args, "i|i", &data->start, &data->stop)) { - fm_remove_filter(); + FmData.count = 0; + FmData.stop = 0; + if (!PyArg_ParseTuple(args, "i|i", &FmData.start, &FmData.stop)) { return NULL; } fm_setup_hooks(); Py_RETURN_NONE; - -error: - PyErr_SetString(PyExc_RuntimeError, "PyMem_RawMalloc() returns NULL"); - return NULL; } static PyObject* remove_mem_hooks(PyObject *self) { fm_remove_hooks(); - fm_remove_filter(); Py_RETURN_NONE; } From 22ab19a42b110d87d1bf88ebf7eeea992c94bec9 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Fri, 30 Jun 2017 11:25:02 +0200 Subject: [PATCH 3/3] Add the Misc/NEWS entry. --- Misc/NEWS.d/next/Tests/2017-06-30-11-20-20.bpo-30695.lo7FQX.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2017-06-30-11-20-20.bpo-30695.lo7FQX.rst diff --git a/Misc/NEWS.d/next/Tests/2017-06-30-11-20-20.bpo-30695.lo7FQX.rst b/Misc/NEWS.d/next/Tests/2017-06-30-11-20-20.bpo-30695.lo7FQX.rst new file mode 100644 index 00000000000000..a57bbe73c27b13 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-06-30-11-20-20.bpo-30695.lo7FQX.rst @@ -0,0 +1,2 @@ +Add the `set_nomemory(start, stop)` and `remove_mem_hooks()` functions to +the _testcapi module.