Skip to content

Commit 35d2ca2

Browse files
vstinnervstinner
authored
[3.6] bpo-30523, bpo-30764, bpo-30776: Sync regrtest from master (#2441)
* bpo-30523: regrtest --list-cases --match (#2401) * regrtest --list-cases now supports --match and --match-file options. Example: ./python -m test --list-cases -m FileTests test_os * --list-cases now also sets support.verbose to False to prevent messages to stdout when loading test modules. * Add support._match_test() private function. (cherry picked from commit ace56d5) * bpo-30764: regrtest: add --fail-env-changed option (#2402) * bpo-30764: regrtest: change exit code on failure * Exit code 2 if failed tests ("bad") * Exit code 3 if interrupted * bpo-30764: regrtest: add --fail-env-changed option If the option is set, mark a test as failed if it alters the environment, for example if it creates a file without removing it. (cherry picked from commit 63f54c6) * bpo-30776: reduce regrtest -R false positives (#2422) * Change the regrtest --huntrleaks checker to decide if a test file leaks or not. Require that each run leaks at least 1 reference. * Warmup runs are now completely ignored: ignored in the checker test and not used anymore to compute the sum. * Add an unit test for a reference leak. Example of reference differences previously considered a failure (leak) and now considered as success (success, no leak): [3, 0, 0] [0, 1, 0] [8, -8, 1] (cherry picked from commit 48b5c42)
1 parent 39e501a commit 35d2ca2

5 files changed

Lines changed: 140 additions & 49 deletions

File tree

‎Lib/test/libregrtest/cmdline.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,9 @@ def _create_parser():
255255
' , don\'t execute them')
256256
group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
257257
help='enable Profile Guided Optimization training')
258+
group.add_argument('--fail-env-changed', action='store_true',
259+
help='if a test file alters the environment, mark '
260+
'the test as failed')
258261

259262
return parser
260263

‎Lib/test/libregrtest/main.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -256,9 +256,13 @@ def _list_cases(self, suite):
256256
if isinstance(test, unittest.TestSuite):
257257
self._list_cases(test)
258258
elif isinstance(test, unittest.TestCase):
259-
print(test.id())
259+
if support._match_test(test):
260+
print(test.id())
260261

261262
def list_cases(self):
263+
support.verbose = False
264+
support.match_tests = self.ns.match_tests
265+
262266
for test in self.selected:
263267
abstest = get_abs_module(self.ns, test)
264268
try:
@@ -474,6 +478,8 @@ def finalize(self):
474478
result = "FAILURE"
475479
elif self.interrupted:
476480
result = "INTERRUPTED"
481+
elif self.environment_changed and self.ns.fail_env_changed:
482+
result = "ENV CHANGED"
477483
else:
478484
result = "SUCCESS"
479485
print("Tests result: %s" % result)
@@ -534,7 +540,13 @@ def _main(self, tests, kwargs):
534540
self.rerun_failed_tests()
535541

536542
self.finalize()
537-
sys.exit(len(self.bad) > 0 or self.interrupted)
543+
if self.bad:
544+
sys.exit(2)
545+
if self.interrupted:
546+
sys.exit(130)
547+
if self.ns.fail_env_changed and self.environment_changed:
548+
sys.exit(3)
549+
sys.exit(0)
538550

539551

540552
def removepy(names):

‎Lib/test/libregrtest/refleak.py‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,21 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
9393
rc_before = rc_after
9494
fd_before = fd_after
9595
print(file=sys.stderr)
96+
9697
# These checkers return False on success, True on failure
9798
def check_rc_deltas(deltas):
98-
return any(deltas)
99+
# bpo-30776: Try to ignore false positives:
100+
#
101+
# [3, 0, 0]
102+
# [0, 1, 0]
103+
# [8, -8, 1]
104+
#
105+
# Expected leaks:
106+
#
107+
# [5, 5, 6]
108+
# [10, 1, 1]
109+
return all(delta >= 1 for delta in deltas)
110+
99111
def check_alloc_deltas(deltas):
100112
# At least 1/3rd of 0s
101113
if 3 * deltas.count(0) < len(deltas):
@@ -104,14 +116,21 @@ def check_alloc_deltas(deltas):
104116
if not set(deltas) <= {1,0,-1}:
105117
return True
106118
return False
119+
120+
def check_fd_deltas(deltas):
121+
return any(deltas)
122+
107123
failed = False
108124
for deltas, item_name, checker in [
109125
(rc_deltas, 'references', check_rc_deltas),
110126
(alloc_deltas, 'memory blocks', check_alloc_deltas),
111-
(fd_deltas, 'file descriptors', check_rc_deltas)]:
127+
(fd_deltas, 'file descriptors', check_fd_deltas)
128+
]:
129+
# ignore warmup runs
130+
deltas = deltas[nwarmup:]
112131
if checker(deltas):
113132
msg = '%s leaked %s %s, sum=%s' % (
114-
test, deltas[nwarmup:], item_name, sum(deltas))
133+
test, deltas, item_name, sum(deltas))
115134
print(msg, file=sys.stderr, flush=True)
116135
with open(fname, "a") as refrep:
117136
print(msg, file=refrep)

‎Lib/test/support/__init__.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1898,6 +1898,23 @@ def _run_suite(suite):
18981898
raise TestFailed(err)
18991899

19001900

1901+
def _match_test(test):
1902+
global match_tests
1903+
1904+
if match_tests is None:
1905+
return True
1906+
test_id = test.id()
1907+
1908+
for match_test in match_tests:
1909+
if fnmatch.fnmatchcase(test_id, match_test):
1910+
return True
1911+
1912+
for name in test_id.split("."):
1913+
if fnmatch.fnmatchcase(name, match_test):
1914+
return True
1915+
return False
1916+
1917+
19011918
def run_unittest(*classes):
19021919
"""Run tests from unittest.TestCase-derived classes."""
19031920
valid_types = (unittest.TestSuite, unittest.TestCase)
@@ -1912,20 +1929,7 @@ def run_unittest(*classes):
19121929
suite.addTest(cls)
19131930
else:
19141931
suite.addTest(unittest.makeSuite(cls))
1915-
def case_pred(test):
1916-
if match_tests is None:
1917-
return True
1918-
test_id = test.id()
1919-
1920-
for match_test in match_tests:
1921-
if fnmatch.fnmatchcase(test_id, match_test):
1922-
return True
1923-
1924-
for name in test_id.split("."):
1925-
if fnmatch.fnmatchcase(name, match_test):
1926-
return True
1927-
return False
1928-
_filter_suite(suite, case_pred)
1932+
_filter_suite(suite, _match_test)
19291933
_run_suite(suite)
19301934

19311935
#=======================================================================

‎Lib/test/test_regrtest.py‎

Lines changed: 83 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -377,19 +377,19 @@ def parse_executed_tests(self, output):
377377
return list(match.group(1) for match in parser)
378378

379379
def check_executed_tests(self, output, tests, skipped=(), failed=(),
380-
omitted=(), randomize=False, interrupted=False):
380+
env_changed=(), omitted=(),
381+
randomize=False, interrupted=False,
382+
fail_env_changed=False):
381383
if isinstance(tests, str):
382384
tests = [tests]
383385
if isinstance(skipped, str):
384386
skipped = [skipped]
385387
if isinstance(failed, str):
386388
failed = [failed]
389+
if isinstance(env_changed, str):
390+
env_changed = [env_changed]
387391
if isinstance(omitted, str):
388392
omitted = [omitted]
389-
ntest = len(tests)
390-
nskipped = len(skipped)
391-
nfailed = len(failed)
392-
nomitted = len(omitted)
393393

394394
executed = self.parse_executed_tests(output)
395395
if randomize:
@@ -415,11 +415,17 @@ def list_regex(line_format, tests):
415415
regex = list_regex('%s test%s failed', failed)
416416
self.check_line(output, regex)
417417

418+
if env_changed:
419+
regex = list_regex('%s test%s altered the execution environment',
420+
env_changed)
421+
self.check_line(output, regex)
422+
418423
if omitted:
419424
regex = list_regex('%s test%s omitted', omitted)
420425
self.check_line(output, regex)
421426

422-
good = ntest - nskipped - nfailed - nomitted
427+
good = (len(tests) - len(skipped) - len(failed)
428+
- len(omitted) - len(env_changed))
423429
if good:
424430
regex = r'%s test%s OK\.$' % (good, plural(good))
425431
if not skipped and not failed and good > 1:
@@ -429,10 +435,12 @@ def list_regex(line_format, tests):
429435
if interrupted:
430436
self.check_line(output, 'Test suite interrupted by signal SIGINT.')
431437

432-
if nfailed:
438+
if failed:
433439
result = 'FAILURE'
434440
elif interrupted:
435441
result = 'INTERRUPTED'
442+
elif fail_env_changed and env_changed:
443+
result = 'ENV CHANGED'
436444
else:
437445
result = 'SUCCESS'
438446
self.check_line(output, 'Tests result: %s' % result)
@@ -604,7 +612,7 @@ def test_failing(self):
604612
test_failing = self.create_test('failing', code=code)
605613
tests = [test_ok, test_failing]
606614

607-
output = self.run_tests(*tests, exitcode=1)
615+
output = self.run_tests(*tests, exitcode=2)
608616
self.check_executed_tests(output, tests, failed=test_failing)
609617

610618
def test_resources(self):
@@ -703,7 +711,7 @@ def test_fromfile(self):
703711
def test_interrupted(self):
704712
code = TEST_INTERRUPTED
705713
test = self.create_test('sigint', code=code)
706-
output = self.run_tests(test, exitcode=1)
714+
output = self.run_tests(test, exitcode=130)
707715
self.check_executed_tests(output, test, omitted=test,
708716
interrupted=True)
709717

@@ -732,7 +740,7 @@ def test_slow_interrupted(self):
732740
args = ("--slowest", "-j2", test)
733741
else:
734742
args = ("--slowest", test)
735-
output = self.run_tests(*args, exitcode=1)
743+
output = self.run_tests(*args, exitcode=130)
736744
self.check_executed_tests(output, test,
737745
omitted=test, interrupted=True)
738746

@@ -772,9 +780,43 @@ def test_run(self):
772780
builtins.__dict__['RUN'] = 1
773781
""")
774782
test = self.create_test('forever', code=code)
775-
output = self.run_tests('--forever', test, exitcode=1)
783+
output = self.run_tests('--forever', test, exitcode=2)
776784
self.check_executed_tests(output, [test]*3, failed=test)
777785

786+
def check_leak(self, code, what):
787+
test = self.create_test('huntrleaks', code=code)
788+
789+
filename = 'reflog.txt'
790+
self.addCleanup(support.unlink, filename)
791+
output = self.run_tests('--huntrleaks', '3:3:', test,
792+
exitcode=2,
793+
stderr=subprocess.STDOUT)
794+
self.check_executed_tests(output, [test], failed=test)
795+
796+
line = 'beginning 6 repetitions\n123456\n......\n'
797+
self.check_line(output, re.escape(line))
798+
799+
line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what)
800+
self.assertIn(line2, output)
801+
802+
with open(filename) as fp:
803+
reflog = fp.read()
804+
self.assertIn(line2, reflog)
805+
806+
@unittest.skipUnless(Py_DEBUG, 'need a debug build')
807+
def test_huntrleaks(self):
808+
# test --huntrleaks
809+
code = textwrap.dedent("""
810+
import unittest
811+
812+
GLOBAL_LIST = []
813+
814+
class RefLeakTest(unittest.TestCase):
815+
def test_leak(self):
816+
GLOBAL_LIST.append(object())
817+
""")
818+
self.check_leak(code, 'references')
819+
778820
@unittest.skipUnless(Py_DEBUG, 'need a debug build')
779821
def test_huntrleaks_fd_leak(self):
780822
# test --huntrleaks for file descriptor leak
@@ -799,24 +841,7 @@ def test_leak(self):
799841
fd = os.open(__file__, os.O_RDONLY)
800842
# bug: never cloes the file descriptor
801843
""")
802-
test = self.create_test('huntrleaks', code=code)
803-
804-
filename = 'reflog.txt'
805-
self.addCleanup(support.unlink, filename)
806-
output = self.run_tests('--huntrleaks', '3:3:', test,
807-
exitcode=1,
808-
stderr=subprocess.STDOUT)
809-
self.check_executed_tests(output, [test], failed=test)
810-
811-
line = 'beginning 6 repetitions\n123456\n......\n'
812-
self.check_line(output, re.escape(line))
813-
814-
line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test
815-
self.assertIn(line2, output)
816-
817-
with open(filename) as fp:
818-
reflog = fp.read()
819-
self.assertIn(line2, reflog)
844+
self.check_leak(code, 'file descriptors')
820845

821846
def test_list_tests(self):
822847
# test --list-tests
@@ -837,19 +862,28 @@ def test_method2(self):
837862
pass
838863
""")
839864
testname = self.create_test(code=code)
865+
866+
# Test --list-cases
840867
all_methods = ['%s.Tests.test_method1' % testname,
841868
'%s.Tests.test_method2' % testname]
842869
output = self.run_tests('--list-cases', testname)
843870
self.assertEqual(output.splitlines(), all_methods)
844871

872+
# Test --list-cases with --match
873+
all_methods = ['%s.Tests.test_method1' % testname]
874+
output = self.run_tests('--list-cases',
875+
'-m', 'test_method1',
876+
testname)
877+
self.assertEqual(output.splitlines(), all_methods)
878+
845879
def test_crashed(self):
846880
# Any code which causes a crash
847881
code = 'import faulthandler; faulthandler._sigsegv()'
848882
crash_test = self.create_test(name="crash", code=code)
849883
ok_test = self.create_test(name="ok")
850884

851885
tests = [crash_test, ok_test]
852-
output = self.run_tests("-j2", *tests, exitcode=1)
886+
output = self.run_tests("-j2", *tests, exitcode=2)
853887
self.check_executed_tests(output, tests, failed=crash_test,
854888
randomize=True)
855889

@@ -898,6 +932,25 @@ def test_method4(self):
898932
subset = ['test_method1', 'test_method3']
899933
self.assertEqual(methods, subset)
900934

935+
def test_env_changed(self):
936+
code = textwrap.dedent("""
937+
import unittest
938+
939+
class Tests(unittest.TestCase):
940+
def test_env_changed(self):
941+
open("env_changed", "w").close()
942+
""")
943+
testname = self.create_test(code=code)
944+
945+
# don't fail by default
946+
output = self.run_tests(testname)
947+
self.check_executed_tests(output, [testname], env_changed=testname)
948+
949+
# fail with --fail-env-changed
950+
output = self.run_tests("--fail-env-changed", testname, exitcode=3)
951+
self.check_executed_tests(output, [testname], env_changed=testname,
952+
fail_env_changed=True)
953+
901954

902955
if __name__ == '__main__':
903956
unittest.main()

0 commit comments

Comments
 (0)