diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 139f066925..26aae2e740 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -391,6 +391,10 @@ jobs: # JUnit file of its own beside junit.xml. # - `timeout` sends SIGINT at 110 minutes, under the 120-minute cap, so # pytest still writes the junit XML, and exits 124; SIGKILL 3 minutes on. + # #2020: GNU's `timeout`, which Ubuntu 26.04 installs as `gnutimeout`. + # Its `timeout` is uutils' (0.10.0), whose `--kill-after` sends SIGKILL + # with the SIGINT, and to its child alone, so a run there exits 137 + # with no teardown and leaves xdist's workers running. # - `prlimit --fsize`: no file over 2 GiB. # - A transient scope, 12G and no swap, at a fifth of the runner's CPU # weight, where `sudo -n` can start one; otherwise uncapped. A scope's @@ -470,7 +474,9 @@ jobs: if [ -n "$SHARDS" ]; then set -- /usr/bin/python3 util/pytest_shards.py "$SHARDS" -- "$@" fi - set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m "$@" + # #2020: GNU's `timeout` on any image, `gnutimeout` where it exists. + gnu_timeout=$(command -v gnutimeout || command -v timeout) + set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 110m "$@" scoped() { unit=$1 shift diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index cfd70cdff6..3bbad37ff3 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -947,7 +947,9 @@ warnings: timings`` takes them all. 3.14t's leg runs pytest once, as before. * ``timeout --signal=INT --kill-after=3m 110m`` ends pytest under the step's 120-minute cap, so it still writes the junit XML, and exits 124; ``prlimit --fsize=2147483648`` - caps any file at 2 GiB. + caps any file at 2 GiB. The ``timeout`` is GNU's, ``gnutimeout`` where it exists + (#2020): on Ubuntu 26.04 ``timeout`` is uutils', whose ``--kill-after`` sends SIGKILL + with the SIGINT, and to its child alone. * Where the scope starts ``true`` under a throwaway unit, ``pcapkit-probe-.scope``, with every flag the tests get (``sudo -n --preserve-env``, ``MemoryMax=12G``, ``MemorySwapMax=0``, ``CPUWeight=20``, ``--uid``, ``--gid``), the tests run in the diff --git a/tests/project/test_gnu_timeout_2020.py b/tests/project/test_gnu_timeout_2020.py new file mode 100644 index 0000000000..0e67d4f8f3 --- /dev/null +++ b/tests/project/test_gnu_timeout_2020.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +"""#2020: every workflow ``timeout`` that needs GNU's semantics runs GNU's. + +Ubuntu 26.04, which ``ubuntu-latest`` now serves, ships uutils coreutils 0.10.0 +as ``timeout`` and GNU coreutils 9.7 as ``gnutimeout``. uutils' ``--kill-after`` +sends SIGKILL with the first signal, and to its child alone: in ``ubuntu:26.04``, +``timeout --signal=INT --kill-after=3s 1 sleep 30`` exits 137 after 1 s where +``gnutimeout`` exits 124, and a grandchild that ignores SIGINT outlives uutils' +kill. ``Run unit tests`` in experimental.yml relies on that grace for pytest's +teardown, so it resolves ``gnutimeout`` where it exists and ``timeout`` otherwise. + +A ``timeout`` with no option, ``timeout 60 apt-get update`` as the apt steps run +it, behaves alike under both in the same image: exit 124, and its child's own +children end with it. So the rule here is that a ``timeout`` given any option +runs through that resolution, in every workflow. + +""" + +from __future__ import annotations + +import os +import pathlib +import re +import shutil +import sys +import tempfile +import unittest +from typing import TYPE_CHECKING + +from tests import _workflows +from tests.project.test_resource_monitor_1740 import DEFAULT, TESTS, sh, step + +if TYPE_CHECKING: + from typing import Iterator + +#: How the tests step resolves GNU's ``timeout``, and how it runs it. +RESOLVE = 'gnu_timeout=$(command -v gnutimeout || command -v timeout)\n' +RUN = '"$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"\n' +#: ``timeout`` run as a command and given an option, which only GNU's honours as GNU does. +OPTIONED = re.compile(r'(? 'Iterator[tuple[str, str, str]]': + """Every ``run:`` script of every workflow, as (file, step, script with comments stripped).""" + for path in sorted(_workflows.WORKFLOWS.glob('*.yml')): + for job_id, job in (_workflows.load(path.name).get('jobs') or {}).items(): + for entry in job.get('steps') or (): + if 'run' in entry: + name = f"{job_id}: {entry.get('name') or entry.get('id') or entry['run'][:40]}" + yield path.name, name, COMMENT.sub(r'\1', entry['run']) + + +class TestNoWorkflowRunsAnOptionedTimeoutDirectly(unittest.TestCase): + + def test_the_pattern(self) -> None: + for text in ('timeout --signal=INT 1 x', 'sudo timeout -k 3 1 x', 'set -- prlimit timeout -s INT 1 x'): + with self.subTest(text=text): + self.assertRegex(text, OPTIONED) + for text in ('"$gnu_timeout" --signal=INT 1 x', 'sudo timeout 60 apt-get update', 'command -v timeout)', + '/usr/bin/gnutimeout -k 1 1 x', 'warn "tests timed out" "timeout sent SIGINT (exit 124)"'): + with self.subTest(text=text): + self.assertNotRegex(text, OPTIONED) + + def test_every_workflow(self) -> None: + seen = 0 + for name, entry, script in scripts(): + seen += 1 + with self.subTest(workflow=name, step=entry): + self.assertIsNone(OPTIONED.search(script), script) + if '$gnu_timeout' in script: + self.assertIn(RESOLVE, script) + self.assertLess(script.index(RESOLVE), script.index('"$gnu_timeout"')) + self.assertTrue(seen) + + def test_a_direct_timeout_fails_it(self) -> None: + script = step(TESTS)['run'] + self.assertIn(RUN, script) + self.assertRegex(script.replace(RUN, 'timeout' + RUN[len('"$gnu_timeout"'):]), OPTIONED) + + def test_the_tests_step_resolves_before_it_runs(self) -> None: + script = step(TESTS)['run'] + self.assertEqual(script.count(RESOLVE), 1) + self.assertEqual(script.count('"$gnu_timeout"'), 1) + self.assertLess(script.index(RESOLVE), script.index(RUN)) + + +@unittest.skipUnless(sys.platform.startswith('linux') and all(map(shutil.which, ('bash', 'timeout', 'prlimit'))), + 'the tests step runs coreutils timeout and util-linux prlimit, as on the Ubuntu runners') +class TestTheTestsStepRunsGNUs(unittest.TestCase): + """The resolution alone, then the whole step, with ``sudo``, ``python`` and ``gnutimeout`` stubbed.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-2020-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.bindir = self.tmp / 'bin' + self.bindir.mkdir() + self.calls = self.tmp / 'calls' + + def stub(self, name: 'str', body: 'str', bindir: 'pathlib.Path | None' = None) -> 'str': + path = (bindir or self.bindir) / name + path.write_text('#!/bin/sh\n' + body, encoding='utf-8') + path.chmod(0o755) + return str(path) + + def resolve(self, path: 'str') -> 'str': + # bash by its own path, since ``path`` holds the stubs alone. + result = sh(RESOLVE + 'echo "$gnu_timeout"\n', {'PATH': path}, (str(shutil.which('bash')), '-e')) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.strip() + + def test_gnutimeout_first_then_timeout(self) -> None: + only = self.tmp / 'only' + only.mkdir() + plain = self.stub('timeout', 'exit 0\n', only) + self.assertEqual(self.resolve(str(only)), plain) + gnu = self.stub('gnutimeout', 'exit 0\n', only) + self.assertEqual(self.resolve(str(only)), gnu) + + def test_the_step_runs_gnutimeout_where_it_exists(self) -> None: + temp = self.tmp / 'temp' + temp.mkdir() + (temp / 'resource-monitor.log').write_text('', encoding='utf-8') + vmstat = self.tmp / 'vmstat' + vmstat.write_text('oom_kill 0\n', encoding='utf-8') + real = shutil.which('gnutimeout') or shutil.which('timeout') + self.stub('gnutimeout', f'echo "gnutimeout $*" >> {self.calls}\nexec {real} "$@"\n') + self.stub('sudo', 'exit 1\n') + self.stub('python', 'exit 0\n') + env = dict(os.environ, RUNNER_TEMP=str(temp), GITHUB_WORKSPACE=str(self.tmp), WORKERS='3', + GRAAL_PYTHON_VM_ARGS='', SHARDS='', LATEST_TIER='', + PATH=str(self.bindir) + os.pathsep + os.environ.get('PATH', '')) + self.assertEqual(shutil.which('sudo', path=env['PATH']), str(self.bindir / 'sudo')) + result = sh(step(TESTS)['run'].replace('/proc/vmstat', str(vmstat)), env, DEFAULT) + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.calls.read_text(encoding='utf-8').splitlines() + self.assertEqual(len(calls), 1, calls) + self.assertTrue(calls[0].startswith('gnutimeout --signal=INT --kill-after=3m 110m python -m pytest '), calls) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_graalpy_shards_1963.py b/tests/project/test_graalpy_shards_1963.py index 9f0500a034..11f12c2000 100644 --- a/tests/project/test_graalpy_shards_1963.py +++ b/tests/project/test_graalpy_shards_1963.py @@ -40,7 +40,7 @@ #: The three lines that build the command the step's caps run, in order. PYTEST = 'set -- python -m pytest "$@"\n' SHARDED = 'set -- /usr/bin/python3 util/pytest_shards.py "$SHARDS" -- "$@"\n' -CAPPED = 'set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m "$@"\n' +CAPPED = 'set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"\n' def rest_ignores(shards: 'str') -> 'list[str]': diff --git a/tests/project/test_resource_monitor_1740.py b/tests/project/test_resource_monitor_1740.py index a3f3113474..74b8b20bc7 100644 --- a/tests/project/test_resource_monitor_1740.py +++ b/tests/project/test_resource_monitor_1740.py @@ -133,6 +133,11 @@ def sh(script: 'str', env: 'dict[str, str]', shell: 'tuple[str, ...]', os.unlink(file.name) +def gnu_timeout(env: 'dict[str, str]') -> 'str | None': + """The ``timeout`` the tests step resolves on ``env``'s ``PATH``: ``gnutimeout`` first (#2020).""" + return shutil.which('gnutimeout', path=env['PATH']) or shutil.which('timeout', path=env['PATH']) + + BASH = ('bash', '--noprofile', '--norc', '-eo', 'pipefail') DEFAULT = ('bash', '-e') @@ -286,7 +291,7 @@ def test_the_timeout_ends_the_tests_before_the_step_cap(self) -> None: script = step(TESTS)['run'] # #1963: the command the timeout runs, pytest or GraalPy's shards of it, is set just before. self.assertIn('set -- python -m pytest "$@"\n', script) - match = re.search(r'timeout --signal=INT --kill-after=(\d+)m (\d+)m "\$@"\n', script) + match = re.search(r'"\$gnu_timeout" --signal=INT --kill-after=(\d+)m (\d+)m "\$@"\n', script) self.assertIsNotNone(match, script) assert match is not None kill_after, duration = map(int, match.groups()) @@ -294,7 +299,7 @@ def test_the_timeout_ends_the_tests_before_the_step_cap(self) -> None: self.assertLess(duration + kill_after, step(TESTS)['timeout-minutes']) def test_no_file_grows_past_2_gib(self) -> None: - self.assertIn(f'set -- prlimit --fsize={FSIZE} timeout ', step(TESTS)['run']) + self.assertIn(f'set -- prlimit --fsize={FSIZE} "$gnu_timeout" ', step(TESTS)['run']) def test_the_scope_is_probed_and_falls_back_with_a_warning(self) -> None: script = step(TESTS)['run'] @@ -398,8 +403,9 @@ def test_the_scope_runs_the_tests_under_the_timeout_and_fsize_cap(self) -> None: self.assertTrue(probe.startswith(SUDO_ARGV.format(unit=unit.group(1), **ids)), probe) self.assertTrue(probe.endswith(f' GRAAL_PYTHON_VM_ARGS={HEAP} true'), probe) self.assertTrue(run.startswith(SUDO_ARGV.format(unit='pcapkit-tests.scope', **ids)), run) - self.assertIn(f' prlimit --fsize={FSIZE} timeout --signal=INT --kill-after=3m 110m python -m pytest -q -n 3 ', - run) + # #2020: GNU's timeout, `gnutimeout` where the image has one, by its path. + self.assertIn(f' prlimit --fsize={FSIZE} {gnu_timeout(self.env)} --signal=INT --kill-after=3m 110m ' + 'python -m pytest -q -n 3 ', run) self.assertIn('args: -m pytest -q -n 3 --dist load', result.stdout) self.assertIn(f'heap {HEAP}\n', result.stdout) self.assertIn(f' GRAAL_PYTHON_VM_ARGS={HEAP} prlimit ', run) diff --git a/tests/project/test_superseded_runs_1938.py b/tests/project/test_superseded_runs_1938.py index 1aabd221c9..7e24a976d6 100644 --- a/tests/project/test_superseded_runs_1938.py +++ b/tests/project/test_superseded_runs_1938.py @@ -758,7 +758,7 @@ def test_a_current_run_keeps_the_tests_status(self) -> None: def test_the_watcher_runs_the_scope_and_names_it(self) -> None: """Every cap stays on pytest, and the watcher runs outside the scope, which it names to ``--scope``.""" script = step(TESTS)['run'] - capped = 'set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m "$@"\n' + capped = 'set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"\n' scoped = 'set -- scoped pcapkit-tests.scope "$@"\n' named = 'watched=pcapkit-tests.scope\n' watched = ('set -- "$WATCHER_PYTHON" util/superseded.py watch --interval 60 --grace 30 --scope "$watched" '