From 7a910c9aa44f1424e676b50f23b74a533b84bf7e Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 23:58:54 -0400 Subject: [PATCH] ci: run GraalPy's experimental tier in shards so no worker outlives its heap (#1963) - util/pytest_shards.py: runs a pytest command once per shard, a directory or a glob of one directory's entries, then once over the rest; each run ignores what earlier shards took and starts fresh xdist workers, writes its own JUnit file, and no later run starts after SIGINT. Standard library only, on /usr/bin/python3. - experimental.yml `experimental` / `Run unit tests`: GraalPy's leg sets SHARDS and hands its pytest command to the script inside the one timeout, file-size cap, scope and watcher; 3.14t runs pytest once, as before. `Upload test timings` takes junit*.xml. - Pins follow the step; tests/project/test_graalpy_shards_1963.py runs the step and the script, and checks the shards partition the tier. - workflows.rst: the shard bullet and the Workflow Scripts row. Local GraalPy 25.4.4 at -n 3: whole tier 3559 s, 8 workers down, heap at the 2 GiB cap; sharded 1879 s, none down, largest heap 1303 MB. --- .github/workflows/experimental.yml | 31 +- docs/source/contributing/workflows.rst | 27 +- tests/project/test_graalpy_shards_1963.py | 269 ++++++++++++++++++ tests/project/test_resource_monitor_1740.py | 4 +- tests/project/test_superseded_runs_1938.py | 5 +- .../test_sweeps_on_latest_only_1732.py | 14 +- util/pytest_shards.py | 167 +++++++++++ 7 files changed, 508 insertions(+), 9 deletions(-) create mode 100644 tests/project/test_graalpy_shards_1963.py create mode 100644 util/pytest_shards.py diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 091291766..139f06692 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -375,6 +375,20 @@ jobs: # is 4 a worker, 12 at -n 3. # Five corekit files at `--vm.Xmx450m -n 1`: exit 3 after 28 passed # without the flag; with it, 3 workers replaced and the run finished. + # - #1963: GraalPy's tier runs in shards, so no worker outlives its heap. + # Job 118912680486, the curated `ALL` tier's first full run, lost nine + # workers at 2.5-3.0 GB RSS, 9-17 minutes into their lives, each + # failing a test that passes alone. So util/pytest_shards.py runs + # pytest once per shard `SHARDS` names, a directory or a glob of one + # directory's entries, then once over the rest, and every run starts + # its workers afresh. Each shard is sized by its largest live heap + # after a full GC, measured locally on GraalPy 25.4.4 at `-n 3`: run + # whole, the tier lost 8 workers, its heap reached the 2 GiB cap, and + # it took 3559 s; in these shards none went down, no heap passed + # 1303 MB, and it took 1879 s. The script runs inside the timeout, + # scope and watcher below, so they hold over the whole tier; on the + # timeout's SIGINT no later run starts. Each shard's run writes a + # JUnit file of its own beside junit.xml. # - `timeout` sends SIGINT at 110 minutes, under the 120-minute cap, so # pytest still writes the junit XML, and exits 124; SIGKILL 3 minutes on. # - `prlimit --fsize`: no file over 2 GiB. @@ -428,6 +442,12 @@ jobs: env: GRAAL_PYTHON_VM_ARGS: ${{ startsWith(matrix.python-version, 'graalpy') && '--vm.Xmx2g --vm.XX:+ExitOnOutOfMemoryError' || '' }} WORKERS: ${{ startsWith(matrix.python-version, 'graalpy') && '3' || 'auto' }} + # #1963: folded, so each line break reads as the space between two shards. + SHARDS: >- + ${{ startsWith(matrix.python-version, 'graalpy') && 'tests/const + tests/corekit/test_fields_n* tests/corekit/test_[a-h]* tests/corekit/test_i* tests/corekit + tests/foundation tests/protocols/internet tests/protocols/[a-m]* tests/protocols + tests/toolkit' || '' }} GH_TOKEN: ${{ github.token }} WATCHER: ${{ steps.watcher.outcome }} WATCHER_PYTHON: ${{ steps.watcher-python.outputs.python-path }} @@ -445,7 +465,12 @@ jobs: for module in $LATEST_TIER; do set -- "$@" --ignore="$module" done - set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m python -m pytest "$@" + set -- python -m pytest "$@" + # #1963: GraalPy's tier in runs that each start fresh workers, under the one cap below. + if [ -n "$SHARDS" ]; then + set -- /usr/bin/python3 util/pytest_shards.py "$SHARDS" -- "$@" + fi + set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m "$@" scoped() { unit=$1 shift @@ -526,14 +551,14 @@ jobs: # As `test`'s, under a name of its own: ci-timing.yml reads `test`'s # `junit-unit-*` files alone, and this one is the record of GraalPy's - # runtime. + # runtime. The glob takes GraalPy's JUnit file per shard too (#1963). - name: Upload test timings if: ${{ !cancelled() && steps.install.outcome == 'success' }} continue-on-error: true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: junit-experimental-${{ matrix.python-version }} - path: ${{ runner.temp }}/junit.xml + path: ${{ runner.temp }}/junit*.xml retention-days: 14 # #1740: what `tests-status` and the monitor's last sample say about the diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 856d20ec8..b215aaaae 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -927,6 +927,24 @@ warnings: is spent exits, xdist fails the test it ran as crashed and starts another, and ``--max-worker-restart=30`` allows 30 restarts in all, where three 16-minute heaps need about 21 in 110 minutes; xdist's default is 4 a worker, 12 at ``-n 3``. +* GraalPy's tier runs in shards (`#1963 + `__), so no worker outlives its + heap. In job 118912680486, the first full run of the curated ``ALL`` tier, nine + workers went down at 2.5-3.0 GB RSS, 9-17 minutes into their lives, each failing a + test that passes alone. So on GraalPy's leg ``Run unit tests`` hands its pytest + command to :file:`util/pytest_shards.py`, which runs it once per shard ``SHARDS`` + names -- a directory, or a glob of one directory's entries such as + ``tests/corekit/test_i*`` -- and once more over the rest, and every run starts its + own workers. Each run also ignores what every earlier shard took, so a glob may + precede its own directory, which then takes the remainder. The shards are sized by + each one's largest live heap after a full GC, measured locally on GraalPy 25.4.4 at + ``-n 3``: run whole, the tier lost 8 workers, its heap reached the 2 GiB cap, and it + took 3559 s; in these shards none went down, no heap passed 1303 MB, and it + took 1879 s. The runs share the step's ``timeout``, scope and watcher, so the + caps hold over the whole tier; on ``timeout``'s SIGINT the run in progress writes + its JUnit file and no later one starts. Each shard's run writes a JUnit file of its + own beside ``junit.xml``, such as ``junit-tests-protocols.xml``, and ``Upload test + timings`` takes them all. 3.14t's leg runs pytest once, as before. * ``timeout --signal=INT --kill-after=3m 110m`` ends pytest under the step's 120-minute cap, so it still writes the junit XML, and exits 124; ``prlimit --fsize=2147483648`` caps any file at 2 GiB. @@ -1155,12 +1173,17 @@ does; each has tests under :file:`tests/project/` that run it. ``experimental.yml``. * - :file:`util/coverage_report.py` - ``Report coverage`` in ``unit-tests.yml``'s ``test``, on the coverage leg. + * - :file:`util/pytest_shards.py` + - ``Run unit tests`` in ``experimental.yml``'s ``experimental``, on GraalPy's leg + (`#1963 `__). The summaries run on the image's ``/usr/bin/python3`` rather than the leg's interpreter, which may have failed to set up or be too broken to run anything, so they need only the standard library, as ``classify_changes.py`` does on the -runner's ``python3``. Both ``changes`` jobs check out on every event, since the -script is in the tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of +runner's ``python3``. :file:`util/pytest_shards.py` runs on ``/usr/bin/python3`` +too, standard library only, so on GraalPy's leg the leg's interpreter runs pytest +alone. Both ``changes`` jobs check out on every event, since the script is in the +tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of the KTLO summary's shell until its shell moves to scripts too; a project pin runs that copy and the script on the same steps and requires the same output. diff --git a/tests/project/test_graalpy_shards_1963.py b/tests/project/test_graalpy_shards_1963.py new file mode 100644 index 000000000..9f0500a03 --- /dev/null +++ b/tests/project/test_graalpy_shards_1963.py @@ -0,0 +1,269 @@ +# -*- coding: utf-8 -*- +"""#1963: GraalPy's leg runs its tier in shards, so no xdist worker outlives its heap. + +#1753 gives every GraalPy process a 2 GiB heap and an exit once it is spent, and a +worker's live heap grows with every test it runs. Over the curated ``ALL`` tier +that took job 118912680486's workers to 2.5-3.0 GB RSS and nine of them down, +each failing the test it ran, though each such test passes alone. So +``Run unit tests`` hands GraalPy's pytest command to :file:`util/pytest_shards.py`, +which runs it once per directory ``SHARDS`` names and once over the rest, each +run with workers of its own, all under the step's one ``timeout``, scope and +watcher. 3.14t's leg runs pytest once, as before. + +:class:`TestTheStep` reads the step, :class:`TestTheStepRuns` runs it with +``sudo`` and ``python`` stubbed on ``PATH``, and :class:`TestTheScript` runs the +script alone. + +""" + +from __future__ import annotations + +import os +import pathlib +import shutil +import signal +import subprocess # nosec: B404 +import sys +import tempfile +import unittest + +from tests._tiers import TIER_ALL, latest_tier_paths, module_tier +from tests.project.test_python_matrix_tiers_1633 import legs +from tests.project.test_python_versions_1660 import ROOT, prose +from tests.project.test_resource_monitor_1740 import DEFAULT, EXPERIMENTAL, HEAP, TESTS, by_impl, sh, step +from util import pytest_shards + +#: GraalPy's shards, in the order they run; the rest of the tier runs last. +SHARDS = ('tests/const tests/corekit/test_fields_n* tests/corekit/test_[a-h]* tests/corekit/test_i* tests/corekit ' + 'tests/foundation tests/protocols/internet tests/protocols/[a-m]* tests/protocols tests/toolkit') +SCRIPT = ROOT / 'util' / 'pytest_shards.py' +#: The three lines that build the command the step's caps run, in order. +PYTEST = 'set -- python -m pytest "$@"\n' +SHARDED = 'set -- /usr/bin/python3 util/pytest_shards.py "$SHARDS" -- "$@"\n' +CAPPED = 'set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m "$@"\n' + + +def rest_ignores(shards: 'str') -> 'list[str]': + """What util/pytest_shards.py's last run, the rest, adds to the command for ``shards``. + + Asked of the script from the root, as the tests step runs it, since a glob shard + resolves against the working directory. + + """ + code = 'import sys; from util import pytest_shards; print(*pytest_shards.runs(sys.argv[1].split(), [])[-1][1])' + result = subprocess.run([sys.executable, '-c', code, shards], cwd=ROOT, capture_output=True, # nosec: B603 + text=True, check=True, timeout=60) + return result.stdout.split() + + +class TestTheStep(unittest.TestCase): + + def test_graalpy_alone_runs_in_shards(self) -> None: + env = step(TESTS)['env'] + versions = [leg['python-version'] for leg in legs(*EXPERIMENTAL)] + self.assertTrue(any(version.startswith('graalpy') for version in versions), versions) + self.assertTrue(any(not version.startswith('graalpy') for version in versions), versions) + for version in versions: + with self.subTest(version=version): + self.assertEqual(by_impl(env['SHARDS'], version), SHARDS if version.startswith('graalpy') else '') + # The heap and the workers are #1753's and #1740's, unchanged. + self.assertEqual(by_impl(env['GRAAL_PYTHON_VM_ARGS'], 'graalpy-25.4'), HEAP) + self.assertEqual(by_impl(env['WORKERS'], 'graalpy-25.4'), '3') + + def test_each_shard_holds_tests_of_the_tier(self) -> None: + for shard in SHARDS.split(): + with self.subTest(shard=shard): + self.assertTrue(shard.startswith('tests/') and not shard.startswith('tests/integration'), shard) + # A directory, or a glob of one directory's entries, as util/pytest_shards.py takes them. + found = sorted(ROOT.glob(shard)) if pytest_shards.MAGIC.search(shard) else [ROOT / shard] + self.assertTrue(found and all(path.exists() for path in found), shard) + modules = [module for path in found + for module in ([path] if path.is_file() else path.rglob('test_*.py'))] + self.assertTrue(any(module_tier(module) == TIER_ALL for module in modules), shard) + self.assertEqual(len(set(SHARDS.split())), len(SHARDS.split())) + + def test_the_shards_run_under_the_one_timeout_scope_and_watcher(self) -> None: + script = step(TESTS)['run'] + self.assertLess(script.index(PYTEST), script.index(SHARDED)) + self.assertLess(script.index(SHARDED), script.index(CAPPED)) + self.assertIn(f'if [ -n "$SHARDS" ]; then\n {SHARDED}fi\n', script) + self.assertLess(script.index(CAPPED), script.index('set -- scoped pcapkit-tests.scope "$@"\n')) + self.assertLess(script.index(CAPPED), script.index('util/superseded.py watch')) + self.assertEqual(script.count('python -m pytest'), 1) + + def test_every_runs_junit_file_is_uploaded(self) -> None: + upload = step('Upload test timings') + self.assertEqual(upload['with']['path'], '${{ runner.temp }}/junit*.xml') + self.assertIn('--junitxml="$RUNNER_TEMP/junit.xml"', step(TESTS)['run']) + + def test_the_docs_say_it(self) -> None: + page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') + for phrase in ('#1963', ':file:`util/pytest_shards.py`', '``SHARDS``', '``junit-tests-protocols.xml``'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, page) + + +@unittest.skipUnless(sys.platform.startswith('linux') and all(map(shutil.which, ('bash', 'timeout', 'prlimit'))), + 'the tests step runs coreutils timeout and util-linux prlimit, as on the Ubuntu runners') +class TestTheStepRuns(unittest.TestCase): + """The tests step with no scope, its ``python`` a stub that records each run's argv.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1963-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.temp = self.tmp / 'temp' + self.temp.mkdir() + self.bindir = self.tmp / 'bin' + self.bindir.mkdir() + (self.temp / 'resource-monitor.log').write_text('', encoding='utf-8') + self.vmstat = self.tmp / 'vmstat' + self.vmstat.write_text('oom_kill 0\n', encoding='utf-8') + self.calls = self.tmp / 'calls' + self.env = dict(os.environ, RUNNER_TEMP=str(self.temp), GITHUB_WORKSPACE=str(ROOT), WORKERS='3', + GRAAL_PYTHON_VM_ARGS=HEAP, SHARDS='tests/a tests/b/c', + PATH=str(self.bindir) + os.pathsep + os.environ.get('PATH', '')) + self.stub('sudo', 'exit 1\n') + + def stub(self, name: 'str', body: 'str') -> 'None': + path = self.bindir / name + path.write_text('#!/bin/sh\n' + body, encoding='utf-8') + path.chmod(0o755) + + def run_step(self, python: 'str', script: 'str | None' = None) -> 'subprocess.CompletedProcess[str]': + """The step, the image's interpreter this one, each run's argv a line of ``self.calls``.""" + self.stub('python', f'echo "$*" >> {self.calls}\n' + python) + self.assertEqual(shutil.which('sudo', path=self.env['PATH']), str(self.bindir / 'sudo')) + script = step(TESTS)['run'] if script is None else script + script = script.replace('/proc/vmstat', str(self.vmstat)) + script = script.replace('/usr/bin/python3 util/pytest_shards.py', f'"{sys.executable}" "{SCRIPT}"') + return sh(script, self.env, DEFAULT) + + def runs(self) -> 'list[list[str]]': + return [line.split() for line in self.calls.read_text(encoding='utf-8').splitlines()] + + def status(self) -> 'dict[str, str]': + text = (self.temp / 'tests-status').read_text(encoding='utf-8') + return dict(line.split('=', 1) for line in text.splitlines()) + + def test_each_shard_then_the_rest_runs_with_workers_of_its_own(self) -> None: + # The second run fails, and a worker goes down in the first and the last. + result = self.run_step(f'n=$(wc -l < {self.calls})\n' + '[ "$n" = 2 ] || echo "[gw0] node down: Not properly terminated"\n' + '[ "$n" = 2 ] && exit 1\nexit 0\n') + self.assertEqual(result.returncode, 1, result.stderr) + first, second, rest = self.runs() + junit = f'--junitxml={self.temp}/junit.xml' + for run in (first, second, rest): + self.assertEqual(run[:9], ['-m', 'pytest', '-q', '-n', '3', '--dist', 'load', + '--max-worker-restart=30', '--durations=30']) + self.assertIn(junit, run) + self.assertEqual(first[-2:], [f'--junitxml={self.temp}/junit-tests-a.xml', 'tests/a']) + self.assertEqual(second[-3:], [f'--junitxml={self.temp}/junit-tests-b-c.xml', '--ignore=tests/a', 'tests/b/c']) + self.assertEqual(rest[-2:], ['--ignore=tests/a', '--ignore=tests/b/c']) + self.assertEqual([arg for arg in rest if arg.startswith('--junitxml')], [junit]) + self.assertNotIn('tests/a', rest) + self.assertIn('pytest_shards: run 3 of 3, the rest', result.stdout) + status = self.status() + self.assertEqual((status['exit'], status['crashed_workers']), ('1', '2')) + + def test_without_shards_pytest_runs_once_as_before(self) -> None: + self.env['SHARDS'] = '' + result = self.run_step('exit 0\n') + self.assertEqual(result.returncode, 0, result.stderr) + (run,) = self.runs() + self.assertEqual(run[:2], ['-m', 'pytest']) + self.assertEqual([arg for arg in run if arg.startswith('--junitxml')], [f'--junitxml={self.temp}/junit.xml']) + self.assertNotIn('pytest_shards', result.stdout) + + def test_the_timeout_ends_the_run_in_progress_and_starts_no_other(self) -> None: + # pytest ends on SIGINT, which timeout sends to its whole process group, with exit 2. + script = step(TESTS)['run'].replace('--kill-after=3m 110m', '--kill-after=5s 1s') + self.assertNotEqual(script, step(TESTS)['run']) + result = self.run_step("trap 'echo interrupted; exit 2' INT\nsleep 30 &\nwait $!\nexit 0\n", script) + self.assertEqual(result.returncode, 124, result.stderr) + self.assertEqual(len(self.runs()), 1) + self.assertIn('interrupted\n', result.stdout) + self.assertIn('pytest_shards: interrupted during tests/a; no later run starts', result.stdout) + self.assertEqual(self.status()['exit'], '124') + + +class TestTheShardsPartitionTheTier(unittest.TestCase): + """GraalPy's runs, collected by pytest through the script: every test of the tier once, none twice.""" + + def test_every_test_runs_once(self) -> None: + collect = [sys.executable, '-m', 'pytest', '--collect-only', '-q', '-p', 'no:cacheprovider', + '--ignore=tests/integration', '--ignore-glob=*_runtime.py', '--ignore-glob=*_regression.py', + *(f'--ignore={module}' for module in latest_tier_paths())] + whole = subprocess.run(collect, cwd=ROOT, capture_output=True, text=True, # nosec: B603 + check=False, timeout=600) + self.assertEqual(whole.returncode, 0, whole.stdout[-2000:] + whole.stderr[-2000:]) + sharded = subprocess.run([sys.executable, str(SCRIPT), SHARDS, '--', *collect], cwd=ROOT, # nosec: B603 + capture_output=True, text=True, check=False, timeout=600) + self.assertEqual(sharded.returncode, 0, sharded.stdout[-2000:] + sharded.stderr[-2000:]) + self.assertNotIn('pytest_shards: skipping', sharded.stdout) + self.assertIn(f'pytest_shards: run {len(SHARDS.split()) + 1} of {len(SHARDS.split()) + 1}, the rest', + sharded.stdout) + tests = [line for line in whole.stdout.splitlines() if '::' in line] + self.assertTrue(tests) + self.assertEqual(sorted(line for line in sharded.stdout.splitlines() if '::' in line), sorted(tests)) + + +class TestTheScript(unittest.TestCase): + + def test_the_runs(self) -> None: + command = ['python', '-m', 'pytest', '-q', '--junitxml=/tmp/t/junit.xml', '--ignore=tests/x'] + self.assertEqual(pytest_shards.runs(['tests/protocols', 'tests/corekit'], command), [ + ('tests/protocols', [*command, '--junitxml=/tmp/t/junit-tests-protocols.xml', 'tests/protocols']), + ('tests/corekit', [*command, '--junitxml=/tmp/t/junit-tests-corekit.xml', '--ignore=tests/protocols', + 'tests/corekit']), + ('the rest', [*command, '--ignore=tests/protocols', '--ignore=tests/corekit']), + ]) + + def test_a_glob_takes_part_of_a_directory_and_the_directory_the_remainder(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1963-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + root = pathlib.Path(tmp.name) / 'd' + (root / 'sub').mkdir(parents=True) + for name in ('test_a.py', 'test_b.py', 'test_c.py', '__init__.py', 'conftest.py', '_helper.py'): + (root / name).write_text('', encoding='utf-8') + a, b, c, sub = (str(root / name) for name in ('test_a.py', 'test_b.py', 'test_c.py', 'sub')) + command = ['pytest'] + # The glob's run collects its directory with the rest of its entries ignored, never the + # package's own files, so pytest's patterns and the command's ignores hold. + self.assertEqual(pytest_shards.runs([f'{root}/test_[ab]*', str(root)], command), [ + (f'{root}/test_[ab]*', [*command, f'--ignore={sub}', f'--ignore={c}', str(root)]), + (str(root), [*command, f'--ignore={a}', f'--ignore={b}', str(root)]), + ('the rest', [*command, f'--ignore={a}', f'--ignore={b}', f'--ignore={root}']), + ]) + self.assertEqual(pytest_shards.junit_for(['--junitxml=t/junit.xml'], 'tests/corekit/test_[a-f]*'), + '--junitxml=t/junit-tests-corekit-test_a-f.xml') + + def test_a_glob_that_selects_nothing_is_skipped(self) -> None: + for shard in ('tests/corekit/test_no_such_module_*', 'tests/*/test_io.py', 'test_*'): + with self.subTest(shard=shard): + self.assertEqual(pytest_shards.runs([shard], ['pytest']), [('the rest', ['pytest'])]) + + def test_a_command_without_a_junit_file_gets_none(self) -> None: + command = ['python', '-m', 'pytest'] + self.assertIsNone(pytest_shards.junit_for(command, 'tests/a')) + self.assertEqual(pytest_shards.runs(['tests/a'], command)[0], ('tests/a', [*command, 'tests/a'])) + + def run_script(self, *args: 'str') -> 'subprocess.CompletedProcess[str]': + return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, # nosec: B603 + check=False, timeout=60) + + def test_a_bad_command_line_is_a_usage_error(self) -> None: + for args in ((), ('tests/a',), ('--', 'true'), ('tests/a', '--')): + with self.subTest(args=args): + self.assertEqual(self.run_script(*args).returncode, pytest_shards.USAGE) + + def test_the_status_is_the_last_failures_and_a_signal_reads_as_a_shells(self) -> None: + self.assertEqual(self.run_script('a b', '--', 'true').returncode, 0) + self.assertEqual(self.run_script('a', '--', 'sh', '-c', 'exit 3', 'sh').returncode, 3) + killed = self.run_script('a', '--', 'sh', '-c', 'kill -TERM $$', 'sh') + self.assertEqual(killed.returncode, 128 + signal.SIGTERM, killed.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_resource_monitor_1740.py b/tests/project/test_resource_monitor_1740.py index 63fba6d93..a3f311347 100644 --- a/tests/project/test_resource_monitor_1740.py +++ b/tests/project/test_resource_monitor_1740.py @@ -284,7 +284,9 @@ def test_graalpy_alone_gets_a_2g_heap_and_3_workers(self) -> None: def test_the_timeout_ends_the_tests_before_the_step_cap(self) -> None: script = step(TESTS)['run'] - match = re.search(r'timeout --signal=INT --kill-after=(\d+)m (\d+)m python -m pytest "\$@"', script) + # #1963: the command the timeout runs, pytest or GraalPy's shards of it, is set just before. + self.assertIn('set -- python -m pytest "$@"\n', script) + match = re.search(r'timeout --signal=INT --kill-after=(\d+)m (\d+)m "\$@"\n', script) self.assertIsNotNone(match, script) assert match is not None kill_after, duration = map(int, match.groups()) diff --git a/tests/project/test_superseded_runs_1938.py b/tests/project/test_superseded_runs_1938.py index 5a2fa57f3..1aabd221c 100644 --- a/tests/project/test_superseded_runs_1938.py +++ b/tests/project/test_superseded_runs_1938.py @@ -198,7 +198,8 @@ def test_the_watcher_runs_on_a_cpython_of_its_own_with_requests(self) -> None: self.assertEqual(tests['env']['WATCHER'], '${{ steps.watcher.outcome }}') self.assertEqual(tests['env']['WATCHER_PYTHON'], '${{ steps.watcher-python.outputs.python-path }}') self.assertIn('set -- "$WATCHER_PYTHON" util/superseded.py watch ', tests['run']) - self.assertNotIn('/usr/bin/python3', tests['run']) + # #1963: the image's interpreter runs GraalPy's shards in the tests step, never the watcher. + self.assertNotIn('/usr/bin/python3 util/superseded.py', tests['run']) # Nothing the leg's interpreter runs installs requests. for entry in legs['steps']: if entry.get('id') != 'watcher': @@ -757,7 +758,7 @@ def test_a_current_run_keeps_the_tests_status(self) -> None: def test_the_watcher_runs_the_scope_and_names_it(self) -> None: """Every cap stays on pytest, and the watcher runs outside the scope, which it names to ``--scope``.""" script = step(TESTS)['run'] - capped = 'set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m python -m pytest "$@"\n' + capped = 'set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m "$@"\n' scoped = 'set -- scoped pcapkit-tests.scope "$@"\n' named = 'watched=pcapkit-tests.scope\n' watched = ('set -- "$WATCHER_PYTHON" util/superseded.py watch --interval 60 --grace 30 --scope "$watched" ' diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 515005a10..f54d955d1 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -43,6 +43,7 @@ from typing import TYPE_CHECKING from tests import _tiers +from tests.project.test_graalpy_shards_1963 import rest_ignores from tests.project.test_python_matrix_tiers_1633 import REQUIRED, is_experimental, job, legs, workflow from tests.project.test_python_versions_1660 import ROOT, live_outputs, prose from tests.project.test_release_gates import NULL, evaluate_expression @@ -171,6 +172,17 @@ def test_the_coverage_command_is_recorded_without_the_real_interpreter(self) -> self.assertEqual(record.read_text(encoding='utf-8').splitlines(), command) +def unsharded(leg: 'dict[str, Any]', where: 'tuple[str, str]', argv: 'list[str]') -> 'list[str]': + """``argv`` less what util/pytest_shards.py's last run adds on a sharded leg (#1963), which it must end with.""" + shards = step_env(leg, where).get('SHARDS', '') + if not shards: + return argv + rest = rest_ignores(shards) + if not rest or argv[-len(rest):] != rest: + raise AssertionError(f'the last run on {leg} does not end with the ignores of {shards!r}: {argv[-60:]}') + return argv[:-len(rest)] + + def ignored(argv: 'list[str]') -> 'list[str]': """The modules the arguments ``--ignore``, less the unit tier's own flags.""" return [arg.split('=', 1)[1] for arg in pytest_args(argv) if arg.startswith('--ignore=') and arg not in TIER_FLAGS] @@ -183,7 +195,7 @@ class TestTheStepSelectsByLeg(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.latest = live_outputs()['latest'] - cls.runs = {leg['python-version']: (leg, run_step(leg, where)) + cls.runs = {leg['python-version']: (leg, unsharded(leg, where, run_step(leg, where))) for where in (JOB, EXPERIMENTAL) for leg in legs(*where)} assert len(cls.runs) == len(legs(*JOB)) + len(legs(*EXPERIMENTAL)), 'a version runs in both jobs' diff --git a/util/pytest_shards.py b/util/pytest_shards.py new file mode 100644 index 000000000..72b54b3b5 --- /dev/null +++ b/util/pytest_shards.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +"""Run a pytest command once per shard, then once over the rest (#1963). + +A GraalPy xdist worker's live heap grows with every test it runs, and +experimental.yml's ``experimental`` job gives each GraalPy process a fixed heap +that ends it once spent (#1753), so a worker that lives through the whole tier +dies in it, and xdist fails the test it ran. Restarting the workers between runs +is what bounds them. ``Run unit tests`` therefore runs, on the GraalPy leg:: + + /usr/bin/python3 util/pytest_shards.py "$SHARDS" -- python -m pytest ... + +``SHARDS`` holds the shards, separated by whitespace, and each is a directory or +a glob of the entries of one directory, such as ``tests/corekit/test_i*``. +The command runs once per shard, then once more over the rest, and each run +starts its own xdist workers: + +* A directory's run appends the directory. +* A glob's run appends the glob's directory and an ``--ignore`` for each of its + entries the glob leaves out, so pytest collects the directory as it would any + other -- its own patterns and the command's ``--ignore`` hold -- where a file + named on the command line would be collected whatever ignores it. A glob that + matches nothing runs nothing and says so, which leaves its tests to the rest. +* Every run but the first also ignores what each earlier shard took, so a glob + of a directory may come before the directory itself, which then takes the + remainder, and a shard an earlier one took whole runs nothing. The last run + ignores every shard. + +Every run gets the command's own options, so its ``-n`` and ``--ignore`` hold in +each; a shard's run also gets a ``--junitxml`` of its own, beside the command's, +named after the shard -- ``junit-tests-protocols.xml`` beside ``junit.xml`` -- +and the last run keeps the command's. + +The runs share the step's one ``timeout``, scope and watcher, which run this +script, so the step's caps hold over the whole tier rather than a shard. On +``SIGINT``, which ``timeout`` sends to its whole process group, the run then in +progress ends as pytest ends it, with its JUnit file written, and no later run +starts. Any other signal ends this script as it would any process. + +The exit status is that of the last run to fail, as a shell gives it for a run +a signal ended, or 0 where none did. The script runs on the image's own +interpreter, apart from the leg's, so it is standard library only. + +""" + +from __future__ import annotations + +import glob +import os +import pathlib +import re +import signal +import subprocess # nosec: B404 +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from types import FrameType + from typing import Optional + +__all__ = ['junit_for', 'main', 'runs'] + +#: Exit status for a bad command line, as :mod:`argparse` uses for a usage error. +USAGE = 2 +#: The option pytest writes its JUnit file by. +JUNIT = '--junitxml=' +#: What makes a shard a glob, as :func:`glob.has_magic` reads it. +MAGIC = re.compile(r'[*?[]') + + +def junit_for(command: 'list[str]', shard: 'str') -> 'Optional[str]': + """The ``--junitxml`` for ``shard``'s run, beside the last one ``command`` has; ``None`` where it has none.""" + found = [arg for arg in command if arg.startswith(JUNIT)] + if not found: + return None + path = pathlib.PurePath(found[-1][len(JUNIT):]) + # A glob's wildcards drop out: ``tests/corekit/test_[a-f]*`` is ``tests-corekit-test_a-f``. + parts = (re.sub(r'[^\w.-]+', '', part) for part in pathlib.PurePath(shard).parts) + slug = '-'.join(part for part in parts if part not in ('', '.')) + return f'{JUNIT}{path.with_name(f"{path.stem}-{slug}{path.suffix}")}' + + +def select(shard: 'str') -> 'tuple[str, list[str], list[str]]': + """The path ``shard``'s run collects, the paths under it that run ignores, and the paths ``shard`` takes. + + Raises: + ValueError: A glob whose directory holds wildcards, or that matches nothing. + + """ + if not MAGIC.search(shard): + return shard, [], [shard] + directory, _ = os.path.split(shard) + if not directory or MAGIC.search(directory) or not os.path.isdir(directory): + raise ValueError(f'{shard}: a glob shard globs the entries of one directory') + taken = sorted(glob.glob(shard)) + if not taken: + raise ValueError(f'{shard}: matches nothing') + # glob() skips dot-entries; a package's own files and its conftest.py are never a test's. + left = sorted(path for path in glob.glob(os.path.join(directory, '*')) + if path not in taken and not os.path.basename(path).startswith('_') + and os.path.basename(path) != 'conftest.py') + return directory, left, taken + + +def runs(shards: 'list[str]', command: 'list[str]') -> 'list[tuple[str, list[str]]]': + """Every run's name and argv, in order: one per shard, then the rest; a shard that selects nothing is skipped.""" + argvs = [] + taken = [] # type: list[str] + for shard in shards: + try: + path, ignores, took = select(shard) + except ValueError as error: + print(f'pytest_shards: skipping {error}', flush=True) + continue + took = [entry for entry in took if entry not in taken] + if not took: + print(f'pytest_shards: skipping {shard}: an earlier shard took it', flush=True) + continue + junit = junit_for(command, shard) + ignored = [*taken, *(entry for entry in ignores if entry not in taken)] + argvs.append((shard, [*command, *([junit] if junit else []), *(f'--ignore={entry}' for entry in ignored), + path])) + taken.extend(took) + argvs.append(('the rest', [*command, *(f'--ignore={entry}' for entry in taken)])) + return argvs + + +def main(argv: 'Optional[list[str]]' = None) -> 'int': + """Run ``SHARDS ... -- COMMAND ...`` as the module docstring says; the exit status.""" + args = sys.argv[1:] if argv is None else argv + if '--' not in args: + print(f'usage: {pathlib.Path(sys.argv[0]).name} SHARDS [SHARDS ...] -- COMMAND [ARGUMENT ...]', + file=sys.stderr) + return USAGE + split = args.index('--') + shards = [shard for arg in args[:split] for shard in arg.split()] + command = args[split + 1:] + if not shards or not command: + print('pytest_shards: name at least one shard, and the command after --', file=sys.stderr) + return USAGE + + interrupted = [] + + def stop(signum: 'int', frame: 'Optional[FrameType]') -> 'None': # pylint: disable=unused-argument + interrupted.append(signum) + + # An ignored SIGINT stays ignored, for this script and so for pytest. + if signal.getsignal(signal.SIGINT) is not signal.SIG_IGN: + signal.signal(signal.SIGINT, stop) + + status = 0 + every = runs(shards, command) + for index, (name, run) in enumerate(every, 1): + print(f'pytest_shards: run {index} of {len(every)}, {name}', flush=True) + code = subprocess.call(run) # nosec: B603 - the step's own command + if code < 0: + # Ended by a signal: the status a shell gives it. + code = 128 - code + if code: + status = code + if interrupted: + print(f'pytest_shards: interrupted during {name}; no later run starts', flush=True) + break + return status + + +if __name__ == '__main__': + sys.exit(main())