From 6a6eea65e31f3b4f9c1ade470d8e1ccf900d7cb0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Oct 2017 15:59:38 +0200 Subject: [PATCH 1/3] bpo-31692: COUNT_ALLOCS now writes into stderr Statistics are now written into stderr, rather than stdout. --- .../Core and Builtins/2017-10-06-21-56-47.bpo-31692.osJuVJ.rst | 2 ++ Python/pythonrun.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-10-06-21-56-47.bpo-31692.osJuVJ.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-10-06-21-56-47.bpo-31692.osJuVJ.rst b/Misc/NEWS.d/next/Core and Builtins/2017-10-06-21-56-47.bpo-31692.osJuVJ.rst new file mode 100644 index 00000000000000..041f166dffebbc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-10-06-21-56-47.bpo-31692.osJuVJ.rst @@ -0,0 +1,2 @@ +COUNT_ALLOCSi debug mode now writes allocations statistics into stderr, +rather than stdout. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 2ffecc722dc317..ec7d9fcf09f835 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -481,7 +481,7 @@ Py_Finalize(void) /* Debugging stuff */ #ifdef COUNT_ALLOCS - dump_counts(stdout); + dump_counts(stderr); #endif PRINT_TOTAL_REFS(); From 2f1e96c09c04ea47ab72b6b0a453fb490665bc20 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 3 Jul 2016 22:27:26 +0300 Subject: [PATCH 2/3] bpo-19527: Fixed tests with defined COUNT_ALLOCS. (cherry picked from commit a793037d803abf098d172f686e2b95d27863c54d) --- Lib/test/support/__init__.py | 3 +++ Lib/test/test_gc.py | 4 +++- Lib/test/test_module.py | 3 ++- Lib/test/test_threading.py | 3 ++- Lib/test/test_weakref.py | 1 + 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ef474e00b68cc3..f4bceb07f65871 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1795,6 +1795,9 @@ def py3k_bytes(b): except TypeError: return bytes(b) +requires_type_collecting = unittest.skipIf(hasattr(sys, 'getcounts'), + 'types are immortal if COUNT_ALLOCS is defined') + def args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current settings in sys.flags.""" diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index ed01c9802fc1ee..7e47b2d3a27bae 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,5 +1,6 @@ import unittest -from test.test_support import verbose, run_unittest, start_threads +from test.support import (verbose, run_unittest, start_threads, + requires_type_collecting) import sys import time import gc @@ -90,6 +91,7 @@ class A: del a self.assertNotEqual(gc.collect(), 0) + @requires_type_collecting def test_newinstance(self): class A(object): pass diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 21eaf3ed721653..e3184198a588d5 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -1,6 +1,6 @@ # Test the module type import unittest -from test.test_support import run_unittest, gc_collect +from test.support import run_unittest, gc_collect, requires_type_collecting import sys ModuleType = type(sys) @@ -65,6 +65,7 @@ def f(): gc_collect() self.assertEqual(f().__dict__["bar"], 4) + @requires_type_collecting def test_clear_dict_in_ref_cycle(self): destroyed = [] m = ModuleType("foo") diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 4c21e6baa76cbe..8f269de68f5c79 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1,7 +1,7 @@ # Very rudimentary test of threading module import test.test_support -from test.test_support import verbose, cpython_only +from test.support import verbose, cpython_only, requires_type_collecting from test.script_helper import assert_python_ok import random @@ -812,6 +812,7 @@ def run(): self.assertIn("ZeroDivisionError", err) self.assertNotIn("Unhandled exception", err) + @requires_type_collecting def test_print_exception_stderr_is_none_1(self): script = r"""if 1: import sys diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 415d5ebbd728bf..ceb513022ce318 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -601,6 +601,7 @@ class D: del c1, c2, C, D gc.collect() + @support.requires_type_collecting def test_callback_in_cycle_resurrection(self): import gc From 343a02ec6fdb2bcf3109fb7c4af21bdd0853c0e9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Oct 2017 15:57:26 +0200 Subject: [PATCH 3/3] bpo-31692: Fix more tests for COUNT_ALLOCS * Fix test_abc: Add @requires_type_collecting to test_cache_leak() * Fix test_hash: get_hash() helper function doesn't redirect stderr to stdout anymore, stderr is now redirected to its own pipe which is ignored. * Fix test_regrtest: Skip test_huntrleaks() if COUNT_ALLOCS is defined. * Fix test_warnings: Skip two tests which rely on stderr. * Fix test_sys.test_objecttypes(): Fix the object size when COUNT_ALLOCS is defined * Fix test_json, test_tools, test_sys: Don't check stderr if COUNT_ALLOCS is defined. * Fix test_threading: Don't check stderr if COUNT_ALLOCS is defined * Fix test_subprocess: Skip tests checking stderr. * Fix test_multiprocessing --- Lib/json/tests/test_tool.py | 8 ++++++-- Lib/test/test_abc.py | 2 ++ Lib/test/test_hash.py | 2 +- Lib/test/test_multiprocessing.py | 4 +++- Lib/test/test_regrtest.py | 1 + Lib/test/test_subprocess.py | 17 +++++++++++++++++ Lib/test/test_sys.py | 21 +++++++++++++++------ Lib/test/test_threading.py | 8 ++++++-- Lib/test/test_tools.py | 10 +++++++--- Lib/test/test_warnings.py | 4 ++++ Lib/test/test_weakref.py | 2 +- 11 files changed, 63 insertions(+), 16 deletions(-) diff --git a/Lib/json/tests/test_tool.py b/Lib/json/tests/test_tool.py index 27dfb84fdbdbf7..48c4bbff6e2017 100644 --- a/Lib/json/tests/test_tool.py +++ b/Lib/json/tests/test_tool.py @@ -56,7 +56,9 @@ def test_infile_stdout(self): infile = self._create_infile() rc, out, err = assert_python_ok('-m', 'json.tool', infile) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) - self.assertEqual(err, b'') + # If COUNT_ALLOCS is defined, don't check stderr + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') def test_infile_outfile(self): infile = self._create_infile() @@ -66,4 +68,6 @@ def test_infile_outfile(self): with open(outfile, "r") as fp: self.assertEqual(fp.read(), self.expect) self.assertEqual(out, b'') - self.assertEqual(err, b'') + # If COUNT_ALLOCS is defined, don't check stderr + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 6a8c3a13274272..31dc422d34d6ae 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -5,6 +5,7 @@ import unittest, weakref from test import test_support +from support import requires_type_collecting import abc from inspect import isabstract @@ -208,6 +209,7 @@ class C(A, B): C() self.assertEqual(B.counter, 1) + @requires_type_collecting def test_cache_leak(self): # See issue #2521. class A(object): diff --git a/Lib/test/test_hash.py b/Lib/test/test_hash.py index c4f7c66ad16e13..56bdd10cce87bf 100644 --- a/Lib/test/test_hash.py +++ b/Lib/test/test_hash.py @@ -158,7 +158,7 @@ def get_hash(self, repr_, seed=None): env.pop('PYTHONHASHSEED', None) cmd_line = [sys.executable, '-c', self.get_hash_command(repr_)] p = subprocess.Popen(cmd_line, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) out, err = p.communicate() out = test_support.strip_python_stderr(out) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index a77bfee6718ee6..e2bfb623745949 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -2610,7 +2610,9 @@ def test_noforkbomb(self): else: rc, out, err = test.script_helper.assert_python_ok(name) self.assertEqual(out.rstrip(), '123') - self.assertEqual(err, '') + # If COUNT_ALLOCS is defined, don't check stderr + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, '') # # Issue 12098: check sys.flags of child matches that for parent diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 264c74d22ba7e4..18e9b0d5af8519 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -493,6 +493,7 @@ def check_leak(self, code, what): self.assertIn(line2, reflog) @unittest.skipUnless(Py_DEBUG, 'need a debug build') + @support.requires_type_collecting def test_huntrleaks(self): # test --huntrleaks code = textwrap.dedent(""" diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index a5a727efd62d43..f0c57d9c08bd6c 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -46,6 +46,12 @@ SETBINARY = '' + +# bpo-31692: If COUNT_ALLOCS is defined, stderr is flooded with allocation +# statistics and so cannot be checked in a reliable way +skip_unless_usable_stderr = test_support.requires_type_collecting + + class BaseTestCase(unittest.TestCase): def setUp(self): # Try to minimize the number of children we have so this test @@ -272,6 +278,7 @@ def test_stdout_fileobj(self): tf.seek(0) self.assertEqual(tf.read(), "orange") + @skip_unless_usable_stderr def test_stderr_pipe(self): # stderr redirection p = subprocess.Popen([sys.executable, "-c", @@ -280,6 +287,7 @@ def test_stderr_pipe(self): self.addCleanup(p.stderr.close) self.assertStderrEqual(p.stderr.read(), "strawberry") + @skip_unless_usable_stderr def test_stderr_filedes(self): # stderr is set to open file descriptor tf = tempfile.TemporaryFile() @@ -291,6 +299,7 @@ def test_stderr_filedes(self): os.lseek(d, 0, 0) self.assertStderrEqual(os.read(d, 1024), "strawberry") + @skip_unless_usable_stderr def test_stderr_fileobj(self): # stderr is set to open file object tf = tempfile.TemporaryFile() @@ -301,6 +310,7 @@ def test_stderr_fileobj(self): tf.seek(0) self.assertStderrEqual(tf.read(), "strawberry") + @skip_unless_usable_stderr def test_stderr_redirect_with_no_stdout_redirect(self): # test stderr=STDOUT while stdout=None (not set) @@ -322,6 +332,7 @@ def test_stderr_redirect_with_no_stdout_redirect(self): self.assertStderrEqual(stderr, b'') # should be empty self.assertEqual(p.returncode, 0) + @skip_unless_usable_stderr def test_stdout_stderr_pipe(self): # capture stdout and stderr to the same pipe p = subprocess.Popen([sys.executable, "-c", @@ -334,6 +345,7 @@ def test_stdout_stderr_pipe(self): self.addCleanup(p.stdout.close) self.assertStderrEqual(p.stdout.read(), "appleorange") + @skip_unless_usable_stderr def test_stdout_stderr_file(self): # capture stdout and stderr to the same open file tf = tempfile.TemporaryFile() @@ -451,6 +463,7 @@ def test_communicate_stdout(self): self.assertEqual(stdout, "pineapple") self.assertEqual(stderr, None) + @skip_unless_usable_stderr def test_communicate_stderr(self): p = subprocess.Popen([sys.executable, "-c", 'import sys; sys.stderr.write("pineapple")'], @@ -459,6 +472,7 @@ def test_communicate_stderr(self): self.assertEqual(stdout, None) self.assertStderrEqual(stderr, "pineapple") + @skip_unless_usable_stderr def test_communicate(self): p = subprocess.Popen([sys.executable, "-c", 'import sys,os;' @@ -525,6 +539,7 @@ def test_communicate_pipe_buf(self): (stdout, stderr) = p.communicate(string_to_write) self.assertEqual(stdout, string_to_write) + @skip_unless_usable_stderr def test_writes_before_communicate(self): # stdin.write before communicate() p = subprocess.Popen([sys.executable, "-c", @@ -1071,6 +1086,7 @@ def test_terminate_dead(self): # Terminating a dead process self._kill_dead_process('terminate') + @skip_unless_usable_stderr def check_close_std_fds(self, fds): # Issue #9905: test that subprocess pipes still work properly with # some standard fds closed @@ -1173,6 +1189,7 @@ def check_swap_fds(self, stdin_no, stdout_no, stderr_no): # When duping fds, if there arises a situation where one of the fds is # either 0, 1 or 2, it is possible that it is overwritten (#12607). # This tests all combinations of this. + @skip_unless_usable_stderr def test_swap_fds(self): self.check_swap_fds(0, 1, 2) self.check_swap_fds(0, 2, 1) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 5baaa352c0ba76..5a9f81e8f38a7b 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -129,7 +129,9 @@ def test_exit(self): rc, out, err = assert_python_ok('-c', 'import sys; sys.exit()') self.assertEqual(rc, 0) self.assertEqual(out, b'') - self.assertEqual(err, b'') + # If COUNT_ALLOCS is defined, don't check stderr + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') # call with integer argument with self.assertRaises(SystemExit) as cm: @@ -157,23 +159,27 @@ def test_exit(self): rc, out, err = assert_python_failure('-c', 'raise SystemExit, 46') self.assertEqual(rc, 46) self.assertEqual(out, b'') - self.assertEqual(err, b'') + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') # ... and normalized rc, out, err = assert_python_failure('-c', 'raise SystemExit(47)') self.assertEqual(rc, 47) self.assertEqual(out, b'') - self.assertEqual(err, b'') + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') # test that the exit machinery handles long exit codes rc, out, err = assert_python_failure('-c', 'raise SystemExit(47L)') self.assertEqual(rc, 47) self.assertEqual(out, b'') - self.assertEqual(err, b'') + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') rc, out, err = assert_python_ok('-c', 'raise SystemExit(0L)') self.assertEqual(rc, 0) self.assertEqual(out, b'') - self.assertEqual(err, b'') + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') def check_exit_message(code, expected, **env_vars): rc, out, err = assert_python_failure('-c', code, **env_vars) @@ -748,7 +754,10 @@ def delx(self): del self.__x # tupleiterator check(iter(()), size('lP')) # type - s = vsize('P2P15Pl4PP9PP11PI' # PyTypeObject + fmt = 'P2P15Pl4PP9PP11PI' + if hasattr(sys, 'getcounts'): + fmt += '3P2P' + s = vsize(fmt + # PyTypeObject '39P' # PyNumberMethods '3P' # PyMappingMethods '10P' # PySequenceMethods diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 8f269de68f5c79..f6cae1c8a62fa9 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -360,7 +360,9 @@ def child(): self.assertEqual(stdout.strip(), "Woke up, sleep function is: ") stderr = re.sub(r"^\[\d+ refs\]", "", stderr, re.MULTILINE).strip() - self.assertEqual(stderr, "") + # If COUNT_ALLOCS is defined, don't check stderr + if not hasattr(sys, 'getcounts'): + self.assertEqual(stderr, "") def test_enumerate_after_join(self): # Try hard to trigger #1703448: a thread is still returned in @@ -437,7 +439,9 @@ def background_thread(evt): """ _, out, err = assert_python_ok("-c", code) self.assertEqual(out, '') - self.assertEqual(err, '') + # If COUNT_ALLOCS is defined, don't check stderr + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, '') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_is_alive_after_fork(self): diff --git a/Lib/test/test_tools.py b/Lib/test/test_tools.py index 57b3ef11fd5969..682f8da7e0b67e 100644 --- a/Lib/test/test_tools.py +++ b/Lib/test/test_tools.py @@ -69,7 +69,9 @@ def test_selftest(self): rc, out, err = assert_python_ok(self.script, '-d', data_path) self.assertEqual(out, b'') - self.assertEqual(err, b'') + # If COUNT_ALLOCS is defined, don't check stderr + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') backup = data_path + '~' self.assertTrue(os.path.exists(backup)) with open(backup) as f: @@ -82,7 +84,8 @@ def test_selftest(self): rc, out, err = assert_python_ok(self.script, '-c', data_path) self.assertEqual(out, b'') - self.assertEqual(err, b'') + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') with open(backup) as f: self.assertEqual(f.read(), clean) with open(data_path) as f: @@ -93,7 +96,8 @@ def test_selftest(self): f.write(broken) rc, out, err = assert_python_ok(self.script, '-r', data_path) self.assertEqual(out, b'') - self.assertEqual(err, b'') + if not hasattr(sys, 'getcounts'): + self.assertEqual(err, b'') with open(backup) as f: self.assertEqual(f.read(), broken) with open(data_path) as f: diff --git a/Lib/test/test_warnings.py b/Lib/test/test_warnings.py index ae081eeee3f75a..7d7db5197afa16 100644 --- a/Lib/test/test_warnings.py +++ b/Lib/test/test_warnings.py @@ -419,6 +419,8 @@ def test_improper_option(self): rc, out, err = assert_python_ok("-Wxxx", "-c", "pass") self.assertIn(b"Invalid -W option ignored: invalid action: 'xxx'", err) + # Don't check stderr when COUNT_ALLOCS is defined + @test_support.requires_type_collecting def test_warnings_bootstrap(self): # Check that the warnings module does get loaded when -W # is used (see issue #10372 for an example of silent bootstrap failure). @@ -575,6 +577,8 @@ def test_filename_none(self): finally: globals_dict['__file__'] = oldfile + # Don't check stderr when COUNT_ALLOCS is defined + @test_support.requires_type_collecting def test_stderr_none(self): rc, stdout, stderr = assert_python_ok("-c", "import sys; sys.stderr = None; " diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index ceb513022ce318..418481dadd8f59 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -601,7 +601,7 @@ class D: del c1, c2, C, D gc.collect() - @support.requires_type_collecting + @test_support.requires_type_collecting def test_callback_in_cycle_resurrection(self): import gc