From 78ee03a599525e8dcdb07c98408a9bb01d4b8f04 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 19:52:03 -0400 Subject: [PATCH 1/2] ci: run the curated ALL tier on PyPy, split into separate pytest runs (#1958, #2005) - `test`/`integration` gain a blocking `pypy3.11` leg and the KTLO jobs a `pypy3.10` leg, from the matrix util/python_versions.py builds; they run the same ALL selection, caps and timeouts as the other non-latest legs. - PyPy workers grow steadily to about 5.8 GB each, so a PyPy leg runs the tier as several pytest invocations (`SHARDS`: tests/protocols, tests/corekit, then the rest), each starting fresh workers. - pcapkit imports on PyPy 8.0: a PyPy-gated aenum workaround for the builtin `NoneType` in pcapkit/utilities/compat.py. - `lxml` is CPython-only in the test extra and Pipfile; it has no PyPy wheel and only LATEST modules use it. - Interpreter-limit failures gated on `sys.implementation.name`. Project pins and docs updated; the touched pins pass. --- .github/workflows/unit-tests.yml | 45 +++++- Pipfile | 2 +- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 42 ++++-- pcapkit/utilities/compat.py | 18 +++ pyproject.toml | 12 +- tests/_dependency_gates.py | 10 +- .../corekit/test_sentinel_py_abc_1960_unit.py | 13 ++ .../test_sentinel_singleton_1842_unit.py | 7 + tests/foundation/test_extraction_ownership.py | 5 +- tests/project/test_ci_timing.py | 7 +- .../test_ktlo_jobs_gate_nothing_1738.py | 10 +- tests/project/test_pypy_worker_memory_2005.py | 137 ++++++++++++++++++ .../project/test_python_matrix_tiers_1633.py | 12 +- tests/project/test_python_versions_1660.py | 25 +++- .../test_python_versions_update_1660.py | 4 +- .../project/test_script_dependencies_1894.py | 10 +- .../test_sweeps_on_latest_only_1732.py | 74 ++++++++-- .../misc/pcap/test_header_frame_unit.py | 7 +- tests/protocols/misc/test_pcapng_unit.py | 7 +- .../test_schema_metaclass_abc_cache_unit.py | 5 +- tests/test_tier_guard.py | 3 +- tests/utilities/test_compat.py | 2 + .../test_compat_aenum_nonetype_1958_unit.py | 89 ++++++++++++ tests/utilities/test_exceptions_excepthook.py | 3 + util/python_versions.py | 34 +++-- 26 files changed, 515 insertions(+), 72 deletions(-) create mode 100644 tests/project/test_pypy_worker_memory_2005.py create mode 100644 tests/utilities/test_compat_aenum_nonetype_1958_unit.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index bbccce5524..622821d145 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -104,10 +104,16 @@ jobs: ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false - # #1660: .github/python-versions.toml's supported CPython tier, through - # the `python-versions` job, as the `python-version` axis; it blocks - # through `required-checks` (#1633). `include` marks one leg - # `coverage: true`. The KTLO legs are `test-ktlo`'s (#1738). + # #1660: .github/python-versions.toml's supported tier, CPython's then + # PyPy's, through the `python-versions` job, as the `python-version` + # axis; it blocks through `required-checks` (#1633). `include` marks one + # leg `coverage: true`. The KTLO legs are `test-ktlo`'s (#1738). + # + # #1958, the owner's ruling E: every implementation runs the same `ALL` + # tier, so PyPy's legs run it as every CPython leg but `latest` does, + # here and in `integration`. A supported PyPy leg is one more leg of a + # job `required-checks` needs, so it blocks with no context of its own + # for the ruleset to name, and a KTLO one is `test-ktlo`'s. # # Exactly one leg runs under coverage, so measuring it adds no leg to # the queue. It is `latest`, the newest supported version, like every @@ -151,7 +157,9 @@ jobs: # clean installs on 3.10-3.14 (manylinux wheels, no build step). pycrate # is pure Python but its distribution is large (~238MB, see the NGAP # extra's comment in pyproject.toml) -- still cheap in wall time, just not - # "a second" cheap. + # "a second" cheap. On PyPy 8.0, which setup-python's `pypy3.11` is, + # PyPI has no Linux wheel of cryptography, so pip builds it with the + # image's Rust and libssl-dev (#1958). # # PyPCAPFile (pypcapfile) is deliberately NOT here. It used to be kept off # every job in this file: on 3.10/3.11 (where its @@ -250,9 +258,22 @@ jobs: # import no pcapkit, so coverage loses nothing. Both lists are asked of # tests/_tiers.py at run time rather than listed, as `integration` asks # for its selection. + # + # #2005: on PyPy an xdist worker keeps what each test leaves behind, so + # its RSS grows with every test it runs, never in a few heavy modules: + # at `-n 4` the `ALL` tier took each worker to 5.4-5.9 GB and them all + # past 20 GB, measured on PyPy 8.0.1, where a 14 GB cap OOM-killed them + # and a 3 GB `PYPY_GC_MAX` raised MemoryError instead. A smaller `-n` + # only hands each worker more tests. So a PyPy leg runs pytest once per + # directory `SHARDS` names, then once more over the rest, ignoring each + # of them, and every run starts its workers afresh: the largest, with + # tests/protocols, peaked at 2.6 GB a worker and 8.5 GB in all, and the + # tier took 383 s rather than 493 s. Each run but the last writes a + # JUnit file of its own, beside junit.xml; the step fails if any run did. env: COVERAGE: ${{ matrix.coverage }} LATEST: ${{ matrix.python-version == needs.python-versions.outputs.latest }} + SHARDS: ${{ startsWith(matrix.python-version, 'pypy') && 'tests/protocols tests/corekit' || '' }} run: | if [ "$LATEST" = "true" ]; then if ! skipped=$(python -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))"); then @@ -275,6 +296,16 @@ jobs: # Logs every measured process and its parent, for the check below. COVERAGE_DEBUG=pid,process COVERAGE_DEBUG_FILE="$PWD/coverage-processes.log" \ python util/cap_file_size.py python -m coverage run --rcfile=.github/coverage.toml -m pytest "$@" + elif [ -n "$SHARDS" ]; then + status=0 + rest=() + for shard in $SHARDS; do + python util/cap_file_size.py python -m pytest "$@" \ + --junitxml="$RUNNER_TEMP/junit-${shard//\//-}.xml" "$shard" || status=$? + rest+=(--ignore="$shard") + done + python util/cap_file_size.py python -m pytest "$@" "${rest[@]}" || status=$? + exit "$status" else python util/cap_file_size.py python -m pytest "$@" fi @@ -331,13 +362,15 @@ jobs: # newest push runs. One artifact per leg, so the name carries the version. # `continue-on-error` because the file is only diagnostic: an artifact # service outage must not fail a leg that `Required checks passed` needs. + # The glob takes a PyPy leg's JUnit file per run too (#2005); a CPython + # leg writes junit.xml alone. - name: Upload test timings if: ${{ !cancelled() && steps.install.outcome == 'success' }} continue-on-error: true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: junit-unit-${{ matrix.python-version }} - path: ${{ runner.temp }}/junit.xml + path: ${{ runner.temp }}/junit*.xml retention-days: 14 # #1672: the KTLO leg's report, and the last step of every KTLO matrix diff --git a/Pipfile b/Pipfile index e4a690f787..23c6ad1787 100644 --- a/Pipfile +++ b/Pipfile @@ -46,7 +46,7 @@ beautifulsoup4 = {extras = ["html5lib"],version = "*"} requests = {extras = ["socks"],version = "*"} # The markers match the test extra in pyproject.toml (#1894). psutil = {version = "*",markers = "platform_python_implementation == 'CPython'"} -lxml = {version = "*",markers = "platform_python_implementation != 'GraalVM'"} +lxml = {version = "*",markers = "platform_python_implementation == 'CPython'"} markdown-it-py = "*" autopep8 = "*" bandit = "*" diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 32a8ef467c..8f6db0e1fc 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1771-1846``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1804-1879``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1614-1640``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1647-1673``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 3bbad37ff3..1e0935e299 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -376,7 +376,7 @@ the same commit several times over. It also skips ``changes``, Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1602-1608``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1635-1641``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -689,7 +689,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1937`` + - ``unit-tests.yml:1970`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -711,22 +711,22 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1937``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:1970``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1849`` mentions the string without defining it. +``unit-tests.yml:1882`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1848-1858``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1881-1891``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, and ``changes`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1936-1992``). Rulesets match check names literally, with no +(``unit-tests.yml:1969-2025``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where @@ -735,7 +735,7 @@ It accepts the gating jobs as ``skipped`` only on a docs-only pull request, wher ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1939``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1972``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -759,7 +759,7 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:367-384``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:400-417``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -785,6 +785,30 @@ runner's ``Process completed with exit code N.`` for a ``run:`` step, and an action's own. And `#1634 `__'s tag check still blocks the ``pypi`` KTLO legs: it fails every leg alike, and masked, the leg would go on to publish. +PyPy is tiered the same way. ``test`` and ``integration`` run its supported +versions after CPython's, and ``test-ktlo`` and ``integration-ktlo`` its KTLO ones, +each leg running the ``ALL`` test tier (`#1958 +`__, see `Test Tiers`_). A +supported PyPy leg blocks through ``Required checks passed``, which needs its job, +so it adds no context to the ruleset. No ``Compat``, ``engine-tests`` or +``pypcap-parity`` leg runs PyPy. On PyPy 8.0, which setup-python's ``pypy3.11`` +is, PyPI has no Linux wheel of |cryptography|_, so the install step builds it +with the image's Rust and ``libssl-dev``, and none of ``lxml``, so the ``test`` +extra carries ``lxml`` on CPython only. + +On PyPy an xdist worker's RSS grows with every test it runs, rather than in a few +heavy modules, so a PyPy leg's ``Run unit tests`` runs pytest once per directory its +``SHARDS`` names, ``tests/protocols`` and then ``tests/corekit``, and once more over +the rest, ignoring each (`#2005 `__). +Each run starts its workers afresh. Measured on PyPy 8.0.1 at ``-n 4``, one run +over the whole ``ALL`` tier took each worker to 5.4-5.9 GB and the workers together +past 20 GB, beyond the runner's 16 GB; split, the largest run peaked at 2.6 GB a +worker and 8.5 GB in all, and the tier took 383 s rather than 493 s. A smaller +``-n`` would only hand each worker more tests, and a ``PYPY_GC_MAX`` heap cap turns +the kills into ``MemoryError``\ s. Each run but the last writes a JUnit file of its +own beside ``junit.xml``, which ``Upload test timings`` takes too, and the step +fails if any run did. + 3.7 can neither install the ``test`` extra nor import the package, so each 3.7 leg fails one step, with a warning saying why -- ``test`` and ``integration`` at the install step, the ``Compat`` leg at ``Verify package`` -- and concludes @@ -1049,7 +1073,7 @@ Which legs run which tier: - ``ALL`` and ``LATEST``, less the project pins * - ``project-tests`` - the project pins - * - ``test`` off ``latest``, ``test-ktlo``, ``experimental`` + * - ``test`` off ``latest``, PyPy's among them, ``test-ktlo``, ``experimental`` - ``ALL`` * - ``integration`` on ``latest`` - ``ALL`` and ``LATEST`` diff --git a/pcapkit/utilities/compat.py b/pcapkit/utilities/compat.py index c0d0ef3cdd..cf5720c747 100644 --- a/pcapkit/utilities/compat.py +++ b/pcapkit/utilities/compat.py @@ -24,6 +24,24 @@ 'Mapping', 'Tuple', 'List', 'Dict', 'TypeAlias', 'UnionType', ] +# PyPy 8.0 binds ``NoneType`` in :mod:`builtins`, which CPython does not, and +# its own stdlib relies on it (``_sqlite3``, which aenum imports). aenum 3.1.17's +# ``_common`` binds a ``NoneType`` of its own only on a ``NameError``, so there it +# binds none, and the ``from ._common import *`` in aenum's ``__init__`` fails on +# the name ``__all__`` lists: ``AttributeError: module 'aenum._common' has no +# attribute 'NoneType'``. That left pcapkit unimportable on PyPy 8.0 (#1958). +# ``aenum._common`` itself ran to its end and stays in :data:`sys.modules`, so +# binding the name there and importing aenum again completes it, leaving +# :mod:`builtins` as PyPy has it. This module runs before any of pcapkit's that +# imports aenum: pcapkit/__init__.py reaches it through +# pcapkit.utilities.exceptions, ahead of pcapkit.corekit. +if sys.implementation.name == 'pypy': + try: + import aenum # pylint: disable=unused-import + except AttributeError: + sys.modules['aenum._common'].NoneType = type(None) # type: ignore[attr-defined] + import aenum # pylint: disable=unused-import + if sys.version_info < (3, 6): class ModuleNotFoundError(ImportError): # pylint: disable=redefined-builtin """Module not found.""" diff --git a/pyproject.toml b/pyproject.toml index 2276a6c895..519ccf858c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -379,11 +379,13 @@ test = [ # saved README. A pure-Python wheel; unpinned, like the entries above. "markdown-it-py", # The strict XML parser util/ci_timing.py reads JUnit with and - # util/pyshark_encap_map.py reads PDML with (#1894). PyPI has wheels for - # CPython and PyPy but none for GraalPy, which reports its implementation as - # ``GraalVM``; the tests that parse skip there, and the scripts do not fall - # back to a laxer parser. Unpinned, like the entries above. - "lxml; platform_python_implementation != 'GraalVM'", + # util/pyshark_encap_map.py reads PDML with (#1894). CPython only, like + # psutil: PyPI has no wheel for GraalPy, nor for PyPy 8.0, which + # setup-python's ``pypy3.11`` is, and the image has no libxml2 headers to + # build one (#1958). The tests that parse skip there, all of them on + # ``latest`` alone, and the scripts do not fall back to a laxer parser. + # Unpinned, like the entries above. + "lxml; platform_python_implementation == 'CPython'", ] # setuptools specific diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index e548cc3552..b85952d5d4 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -262,10 +262,12 @@ # The strict XML parser ``HAS_LXML`` asks for (#1894): util/ci_timing.py # reads JUnit and util/pyshark_encap_map.py reads PDML with it, and neither # falls back to a laxer parser. The ``test`` extra carries it with a - # ``platform_python_implementation != 'GraalVM'`` marker, because PyPI has - # no wheel for GraalPy, so the gate skips on experimental.yml's GraalPy leg. - # That is no :data:`DEPENDENCY_GATE_EXCLUSIONS` entry: this module does not - # model markers and does not read experimental.yml, so every job it reads + # ``platform_python_implementation == 'CPython'`` marker, because PyPI has + # no wheel for GraalPy or PyPy 8.0, so the gate skips on experimental.yml's + # GraalPy leg and on unit-tests.yml's PyPy legs (#1958). That is no + # :data:`DEPENDENCY_GATE_EXCLUSIONS` entry: this module does not model + # markers, every module gated on it is ``LATEST``, which no PyPy leg runs, + # and this module does not read experimental.yml, so every job it reads # satisfies the gate, and an entry would describe a gap that is not there. 'lxml': ('lxml',), } diff --git a/tests/corekit/test_sentinel_py_abc_1960_unit.py b/tests/corekit/test_sentinel_py_abc_1960_unit.py index be528b7f66..75b5e6416c 100644 --- a/tests/corekit/test_sentinel_py_abc_1960_unit.py +++ b/tests/corekit/test_sentinel_py_abc_1960_unit.py @@ -18,6 +18,12 @@ trusting the block. :class:`CModuleTests` pins that, under the C module, the names let through for ``_py_abc`` are still refused. +Neither runs on PyPy (#1958). Its built-in ``_abc`` keeps ``_py_abc``'s caches +on the class, so a sentinel type there holds both names, and it imports +``_collections_abc`` at startup even under ``-S``, so a child that blocks +``_abc`` builds :mod:`abc` anew under ABCs already made with the old one, and +``import pcapkit`` fails with a metaclass conflict. + """ from __future__ import annotations @@ -38,6 +44,11 @@ #: interpreter. __tier__ = 'ALL' +#: PyPy's ``_abc`` is no C module and keeps the caches on the class (#1958). +ON_PYPY = sys.implementation.name == 'pypy' +#: Why both classes skip there. +PYPY_REASON = "PyPy's built-in _abc keeps _py_abc's caches on the class, and the fallback cannot be forced" + #: Repository root, put first on the child interpreter's path so that it #: imports this tree rather than whatever :mod:`pcapkit` is installed. REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -145,6 +156,7 @@ def _expected(instance_name: 'str | None') -> 'dict[str, str]': return probes +@unittest.skipIf(ON_PYPY, PYPY_REASON) class PyABCFallbackTests(unittest.TestCase): """Every sentinel type behaves under ``_py_abc`` as it does under ``_abc``.""" @@ -181,6 +193,7 @@ def test_every_probe_behaves_as_under_the_c_module(self) -> 'None': self.assertEqual(actual[probe], outcome) +@unittest.skipIf(ON_PYPY, PYPY_REASON) class CModuleTests(unittest.TestCase): """Under the C ``_abc`` module, the names let through for ``_py_abc`` stay refused.""" diff --git a/tests/corekit/test_sentinel_singleton_1842_unit.py b/tests/corekit/test_sentinel_singleton_1842_unit.py index 3103ec22a7..0d44c2af93 100644 --- a/tests/corekit/test_sentinel_singleton_1842_unit.py +++ b/tests/corekit/test_sentinel_singleton_1842_unit.py @@ -162,6 +162,11 @@ class Derived(base): ('subclass the type', lambda obj: _subclass(type(obj))), ('call the type', lambda obj: type(obj)()), ) +#: The probes whose builtin outcome is PyPy's own (#1958): it raises +#: :exc:`AttributeError` for deleting an existing attribute of an immutable +#: builtin type, where CPython raises :exc:`TypeError`. A sentinel type keeps +#: CPython's, which :class:`TypeImmutabilityTests` pins on every interpreter. +PYPY_OWN = frozenset({'delete an existing type attribute'}) if sys.implementation.name == 'pypy' else frozenset() class _ReimportCase(unittest.TestCase): @@ -256,6 +261,8 @@ def test_every_probe_fails_or_succeeds_as_for_the_builtins(self) -> 'None': module = _module() bad = [] for description, probe in PROBES: + if description in PYPY_OWN: + continue expected = {_raised(lambda: probe(obj)) for obj in BUILTINS} # pylint: disable=cell-var-from-loop self.assertEqual(len(expected), 1, (description, expected)) for name, _, _ in SENTINELS: diff --git a/tests/foundation/test_extraction_ownership.py b/tests/foundation/test_extraction_ownership.py index c62a8dce0a..f612ba76bb 100644 --- a/tests/foundation/test_extraction_ownership.py +++ b/tests/foundation/test_extraction_ownership.py @@ -80,7 +80,10 @@ #: callbacks queued by then (graalpython ``GcModuleBuiltins.javaCollect``, at #: graal-25.4.4), so on a loaded runner the finaliser can run after it returns #: (#1963). A finaliser that never runs still fails once the grace is spent. -FINALISER_GRACE = 10.0 if sys.implementation.name == 'graalpy' else 0.0 +#: PyPy frees the extractor, and so runs the finaliser, one collection later +#: than CPython: the class still defines ``__del__`` (#1958, measured on PyPy +#: 8.0.1), so a short grace lets a second collection run there. +FINALISER_GRACE = {'graalpy': 10.0, 'pypy': 1.0}.get(sys.implementation.name, 0.0) def collect_until(done: 'Callable[[], bool]', grace: float = FINALISER_GRACE) -> bool: diff --git a/tests/project/test_ci_timing.py b/tests/project/test_ci_timing.py index 6809537d4f..5d7d276276 100644 --- a/tests/project/test_ci_timing.py +++ b/tests/project/test_ci_timing.py @@ -1118,14 +1118,17 @@ def test_every_unit_and_integration_leg_uploads_its_timings(self) -> 'None': self.assertEqual(names, ['junit-unit-${{ matrix.python-version }}', 'junit-integration-${{ matrix.python-version }}']) self.assertIn(ci_timing.JUNIT_ARTIFACT.format(leg='${{ matrix.python-version }}'), names) - for block in blocks: + # #2005: a PyPy unit leg writes a JUnit file per pytest run, junit.xml among them. + paths = ('junit*.xml', 'junit.xml') + self.assertEqual(len(blocks), len(paths)) + for block, path in zip(blocks, paths): with self.subTest(step=block.split('\n', 3)[:3]): # A failed upload of a diagnostic file must not fail a required leg. # It runs after failed tests too, but not after a failed install # (#1633), which leaves no file to upload. self.assertIn('continue-on-error: true', block) self.assertIn("if: ${{ !cancelled() && steps.install.outcome == 'success' }}", block) - self.assertIn('path: ${{ runner.temp }}/junit.xml', block) + self.assertIn(f'path: ${{{{ runner.temp }}}}/{path}\n', block) # tests/_dependency_gates.py reads exactly one pytest step per job. self.assertNotIn('python -m pytest', block) diff --git a/tests/project/test_ktlo_jobs_gate_nothing_1738.py b/tests/project/test_ktlo_jobs_gate_nothing_1738.py index efe9ffa422..d1107b876b 100644 --- a/tests/project/test_ktlo_jobs_gate_nothing_1738.py +++ b/tests/project/test_ktlo_jobs_gate_nothing_1738.py @@ -27,7 +27,8 @@ import unittest from typing import TYPE_CHECKING -from tests.project.test_python_matrix_tiers_1633 import KTLO, KTLO_GUARD, PARITY, REQUIRED, SUPPORTED +from tests.project.test_python_matrix_tiers_1633 import (KTLO, KTLO_GUARD, PARITY, REQUIRED, SUPPORTED, TESTED_KTLO, + TESTED_SUPPORTED) from tests.project.test_python_versions_1660 import legs as expand from tests.project.test_python_versions_1660 import matrix, workflow @@ -41,9 +42,10 @@ TWINS = {'test': 'test-ktlo', 'integration': 'integration-ktlo', 'pypcap-parity': 'pypcap-parity-ktlo'} #: Each gating job's check-name prefix, as main named its legs before the split. PREFIX = {'test': 'Python', 'integration': 'Integration Python', 'pypcap-parity': 'PyPCAP/PyPCAPFile parity Python'} -#: The KTLO versions each twin runs; ``pypcap-parity`` runs on ``ubuntu-latest`` and below 3.12 only. -KTLO_OF = {'test': KTLO, 'integration': KTLO, 'pypcap-parity': tuple(v for v in PARITY if v in KTLO)} -SUPPORTED_OF = {'test': SUPPORTED, 'integration': SUPPORTED, +#: The KTLO versions each twin runs; ``pypcap-parity`` runs on ``ubuntu-latest`` and below 3.12 only, and +#: on CPython alone, where ``test`` and ``integration`` run PyPy's too (#1958). +KTLO_OF = {'test': TESTED_KTLO, 'integration': TESTED_KTLO, 'pypcap-parity': tuple(v for v in PARITY if v in KTLO)} +SUPPORTED_OF = {'test': TESTED_SUPPORTED, 'integration': TESTED_SUPPORTED, 'pypcap-parity': tuple(v for v in PARITY if v in SUPPORTED)} diff --git a/tests/project/test_pypy_worker_memory_2005.py b/tests/project/test_pypy_worker_memory_2005.py new file mode 100644 index 0000000000..2ad7b7dfcf --- /dev/null +++ b/tests/project/test_pypy_worker_memory_2005.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +"""#2005: a PyPy leg of ``test`` runs the ``ALL`` tier as several pytest runs. + +On PyPy an xdist worker's RSS grows with every test it runs, so through the +whole tier at ``-n 4`` its workers outgrew the runner. ``Run unit tests`` runs +pytest once per directory its ``SHARDS`` names on a PyPy leg, then once more +over the rest, ignoring each, so every run starts its workers afresh. Every +other leg runs pytest once, as before. :class:`TestTheRunsCoverTheTier` runs the +step's own script with a stub ``python``, as +:mod:`tests.project.test_sweeps_on_latest_only_1732` does. + +""" + +from __future__ import annotations + +import os +import pathlib +import shutil +import subprocess # nosec B404 +import sys +import tempfile +import unittest + +from tests import _tiers +from tests.project.test_python_matrix_tiers_1633 import job, legs +from tests.project.test_python_versions_1660 import ROOT, prose +from tests.project.test_sweeps_on_latest_only_1732 import (JOB, RUNNER_TEMP, run_step_each, shards, step, + step_env, unsharded) + +#: Test tier (#1958): checks the workflow's own shell. +__tier__ = 'LATEST' + +#: The ``test`` job's KTLO twin, which runs its steps by YAML alias. +KTLO = ('unit-tests.yml', 'test-ktlo') +#: The directories a PyPy leg runs on their own, in order. +SHARDS = ['tests/protocols', 'tests/corekit'] +#: The page that says how the legs run. +DOC = ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst' + + +def pypy_leg() -> 'dict[str, object]': + """The supported PyPy leg of ``test``, which blocks.""" + found = [leg for leg in legs(*JOB) if str(leg['python-version']).startswith('pypy')] + if len(found) != 1: + raise AssertionError(f'expected one PyPy leg of test, found {found}') + return found[0] + + +class TestOnlyPyPyShards(unittest.TestCase): + + def test_each_pypy_leg_names_the_shards_and_no_other_leg_does(self) -> None: + every = legs(*JOB) + legs(*KTLO) + self.assertTrue([leg for leg in legs(*KTLO) if str(leg['python-version']).startswith('pypy')]) + for leg in every: + with self.subTest(python=leg['python-version']): + pypy = str(leg['python-version']).startswith('pypy') + self.assertEqual(shards(leg), SHARDS if pypy else []) + + def test_the_ktlo_twin_runs_the_same_step(self) -> None: + self.assertEqual(job(*KTLO)['steps'], job(*JOB)['steps']) + + def test_each_shard_is_a_directory_the_tier_keeps(self) -> None: + for shard in SHARDS: + with self.subTest(shard=shard): + self.assertTrue((ROOT / shard).is_dir()) + self.assertTrue([module for module in _tiers.all_tier_paths() if module.startswith(shard + '/')]) + + +@unittest.skipIf(shutil.which('bash') is None, 'the step is a bash script') +class TestTheRunsCoverTheTier(unittest.TestCase): + + def test_a_pypy_leg_runs_each_shard_then_the_rest(self) -> None: + leg = pypy_leg() + runs = run_step_each(leg) + self.assertEqual(len(runs), len(SHARDS) + 1) + common = unsharded(leg, runs) + self.assertEqual(common[:2], ['-m', 'pytest']) + self.assertIn(f'--junitxml={RUNNER_TEMP}/junit.xml', common) + self.assertEqual([run[-1] for run in runs[:-1]], SHARDS) + self.assertEqual(runs[-1][-len(SHARDS):], [f'--ignore={shard}' for shard in SHARDS]) + + def test_every_other_leg_runs_pytest_once(self) -> None: + for leg in legs(*JOB): + if shards(leg): + continue + with self.subTest(python=leg['python-version']): + self.assertEqual(len(run_step_each(leg)), 1) + + def test_a_shard_that_fails_fails_the_step_after_the_rest_ran(self) -> None: + """A failing run does not stop the others, and its status is the step's.""" + with tempfile.TemporaryDirectory() as temp: + stub = pathlib.Path(temp, 'python') + # The first pytest run fails as pytest does on a failed test; `-c` asks the real interpreter. + stub.write_text('#!/bin/sh\ncase "$1" in -c) exec "$REAL_PYTHON" "$@";; ' + 'util/cap_file_size.py) shift; exec "$@";; esac\n' + 'echo run >> "$RUNS"\n' + '[ "$(wc -l < "$RUNS")" -eq 1 ] && exit 1\nexit 0\n', encoding='utf-8') + stub.chmod(0o755) + runs = pathlib.Path(temp, 'runs') + env = dict(os.environ, **step_env(pypy_leg()), RUNS=str(runs), RUNNER_TEMP=temp, + REAL_PYTHON=sys.executable, ANNOTATION_LEVEL='error', + PATH=temp + os.pathsep + os.environ.get('PATH', '')) + env.pop('PYTHONSAFEPATH', None) + result = subprocess.run(['bash', '-e', '-c', step()['run']], env=env, cwd=ROOT, # nosec B603 B607 + capture_output=True, text=True, check=False, timeout=120) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(runs.read_text(encoding='utf-8').count('run'), len(SHARDS) + 1) + + +class TestTheTimingsAreUploaded(unittest.TestCase): + + def test_the_upload_takes_each_runs_junit_file(self) -> None: + upload = next(entry for entry in job(*JOB)['steps'] if entry.get('name') == 'Upload test timings') + self.assertEqual(upload['with']['path'], '${{ runner.temp }}/junit*.xml') + for shard in SHARDS: + with self.subTest(shard=shard): + self.assertIn(f'--junitxml="$RUNNER_TEMP/junit-${{shard//\\//-}}.xml" "$shard"', step()['run']) + + +class TestTheDocsSayIt(unittest.TestCase): + + def test_the_workflows_page_names_the_shards(self) -> None: + page = prose(DOC) + for phrase in ('#2005', '``SHARDS``', '``Run unit tests``', *(f'``{shard}``' for shard in SHARDS)): + with self.subTest(phrase=phrase): + self.assertIn(phrase, page) + + def test_the_step_says_why(self) -> None: + text = prose(ROOT / '.github' / 'workflows' / 'unit-tests.yml') + for phrase in ('#2005:', 'A smaller `-n` only hands each worker more tests.', + 'every run starts its workers afresh'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, text) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_python_matrix_tiers_1633.py b/tests/project/test_python_matrix_tiers_1633.py index ed635730ba..33965dc129 100644 --- a/tests/project/test_python_matrix_tiers_1633.py +++ b/tests/project/test_python_matrix_tiers_1633.py @@ -97,13 +97,17 @@ def tier(impl: 'str', name: 'str') -> 'tuple[str, ...]': #: The job holding experimental legs, the only one (#1660), in a workflow of its own #: since #1732, so that nothing in unit-tests.yml waits on it. EXPERIMENTAL_JOB = ('experimental.yml', 'experimental') +#: What ``test`` and ``integration`` run, and their KTLO twins: CPython's tier +#: then PyPy's, as each implementation runs the ``ALL`` test tier (#1958). +TESTED_SUPPORTED = SUPPORTED + tier('pypy', 'supported') +TESTED_KTLO = KTLO + tier('pypy', 'ktlo') #: Ruleset 23497679's required contexts, which a tier change edits with the file. REQUIRED = frozenset({'Required checks passed'} | {f'Compat Python {v}' for v in SUPPORTED}) #: Every Python matrix, as (workflow, job, the blocking versions it must run). MATRICES = ( - ('unit-tests.yml', 'test', SUPPORTED), - ('unit-tests.yml', 'integration', SUPPORTED), + ('unit-tests.yml', 'test', TESTED_SUPPORTED), + ('unit-tests.yml', 'integration', TESTED_SUPPORTED), ('unit-tests.yml', 'pypcap-parity', tuple(v for v in PARITY if v in SUPPORTED)), ('python-compatibility.yml', 'compatibility', SUPPORTED), ('create-release.yml', 'pypi', SUPPORTED + tier('pypy', 'supported')), @@ -116,8 +120,8 @@ def tier(impl: 'str', name: 'str') -> 'tuple[str, ...]': #: Where each KTLO leg must be, as (workflow, job, KTLO versions). unit-tests.yml's #: are jobs of their own since #1738, which nothing gating waits on. KTLO_LEGS = ( - ('unit-tests.yml', 'test-ktlo', KTLO), - ('unit-tests.yml', 'integration-ktlo', KTLO), + ('unit-tests.yml', 'test-ktlo', TESTED_KTLO), + ('unit-tests.yml', 'integration-ktlo', TESTED_KTLO), ('unit-tests.yml', 'pypcap-parity-ktlo', tuple(v for v in PARITY if v in KTLO)), ('python-compatibility.yml', 'compatibility', KTLO), ('create-release.yml', 'pypi', KTLO_ON_LATEST + tier('pypy', 'ktlo')), diff --git a/tests/project/test_python_versions_1660.py b/tests/project/test_python_versions_1660.py index 08894c1ec0..f940e6bea2 100644 --- a/tests/project/test_python_versions_1660.py +++ b/tests/project/test_python_versions_1660.py @@ -286,6 +286,26 @@ def setups(spec: 'dict[str, Any]') -> 'list[dict[str, Any]]': } +#: #1958, the owner's ruling E: every implementation runs the ``ALL`` tier, so +#: ``test`` and ``integration`` gained PyPy's supported legs, after CPython's, and +#: their KTLO twins PyPy's KTLO ones -- the one move besides ``latest``'s. Read +#: from :data:`TIERS_AT_MAIN`, as each leg is a version the file lists. +PYPY_TESTED = (('unit-tests.yml', 'test'), ('unit-tests.yml', 'integration')) +PYPY_SUPPORTED = [{'python-version': version} + for version, entry in TIERS_AT_MAIN['versions']['pypy'].items() if entry['tier'] == 'supported'] +PYPY_KTLO = [{'python-version': version, 'ktlo': True} + for version, entry in TIERS_AT_MAIN['versions']['pypy'].items() if entry['tier'] == 'ktlo'] + + +def with_pypy(name: 'str', job_id: 'str', expected: 'list[dict[str, Any]]') -> 'list[dict[str, Any]]': + """``expected``, :data:`MAIN`'s legs of a job, with the PyPy legs #1958 added to it.""" + if (name, job_id) not in PYPY_TESTED: + return expected + supported = [leg for leg in expected if leg.get('ktlo') is not True] + ktlo = [leg for leg in expected if leg.get('ktlo') is True] + return supported + [dict(leg) for leg in PYPY_SUPPORTED] + ktlo + [dict(leg) for leg in PYPY_KTLO] + + #: The experimental legs' job name (#1732), which ``test``'s was on those legs from #1660. EXPERIMENTAL_TEMPLATE = 'Python ${{ matrix.python-version }} (experimental)' @@ -551,7 +571,7 @@ def setUpClass(cls) -> None: cls.values = script().outputs(script().load(write_tiers(pathlib.Path(tmp), TIERS_AT_MAIN))) def test_every_matrix_expands_to_mains_legs(self) -> None: - """Same legs, same order; the coverage leg moves from 3.14 to ``latest`` (#1661). + """Same legs, same order; the coverage leg moves from 3.14 to ``latest`` (#1661), and PyPy's join (#1958). Where the clone holds :data:`MAIN`, :data:`MAIN_MATRICES` is read back against it first, so the comparison is with what ``main`` really ran. @@ -570,6 +590,7 @@ def test_every_matrix_expands_to_mains_legs(self) -> None: if (name, job_id) in CONDA_JOBS: # #1822: an Intel macOS leg per version, so osx-64 is built natively. expected = legs(dict(spec, os=CONDA_OS)) + expected = with_pypy(name, job_id, expected) if (name, job_id) in KTLO_TWINS: # #1738: the KTLO legs, in MAIN's order, are the twin job's. twin = KTLO_TWINS[(name, job_id)] @@ -609,7 +630,7 @@ def test_no_job_name_changes_but_the_gates(self) -> None: self.assertEqual(workflow(name)['jobs'][job_id]['name'], template) continue spec = MAIN_MATRICES[(name, job_id)] - before = sorted(render(template, leg) for leg in legs(spec)) + before = sorted(render(template, leg) for leg in with_pypy(name, job_id, legs(spec))) after = sorted(render(workflow(name)['jobs'][job]['name'], leg) for job in (job_id, KTLO_TWINS[(name, job_id)]) for leg in legs(matrix(name, job, self.values))) diff --git a/tests/project/test_python_versions_update_1660.py b/tests/project/test_python_versions_update_1660.py index a0ce75b888..bd520a8007 100644 --- a/tests/project/test_python_versions_update_1660.py +++ b/tests/project/test_python_versions_update_1660.py @@ -409,7 +409,9 @@ def test_the_snapshot_reproduces_the_file_but_for_pypy_312(self) -> None: # Only the tier file: releasing.rst counted the wheels until #1811, so a new PyPy rewrote it. self.assertEqual(sorted(texts), ['.github/python-versions.toml']) self.assertEqual(plan['required'], {'remove': [], 'add': []}) - self.assertEqual(plan['added'], ['PyPI distribution for Python pypy3.12']) + # #1958: a supported PyPy gets a `test` and an `integration` leg too. + self.assertEqual(plan['added'], ['PyPI distribution for Python pypy3.12', + 'Integration Python pypy3.12', 'Python pypy3.12']) def test_as_of_the_file_nothing_changes(self) -> None: """Before PyPy 8.0.0, the sources gave the tier file exactly as #1677 wrote it.""" diff --git a/tests/project/test_script_dependencies_1894.py b/tests/project/test_script_dependencies_1894.py index 1f6cd25d94..5e9352ed7c 100644 --- a/tests/project/test_script_dependencies_1894.py +++ b/tests/project/test_script_dependencies_1894.py @@ -7,10 +7,10 @@ :file:`pyproject.toml` carries them for CI; at the owner's request :file:`Pipfile`'s ``[dev-packages]``, which ``make``'s ``pipenv run`` targets resolve against, carries them too, with the ``test`` extra's markers. Those keep -each off the interpreters PyPI has no wheel of it for: ``psutil`` is CPython -only, and ``lxml`` is everywhere but GraalPy. The ``dev`` extra is not asked: -it holds what the toolchain linting :mod:`pcapkit` must resolve, not the -scripts' dependencies. +each off the interpreters PyPI has no wheel of it for: ``psutil`` and ``lxml`` +are CPython only, ``lxml`` since #1958, as PyPI has none of it for GraalPy or +PyPy 8.0. The ``dev`` extra is not asked: it holds what the toolchain linting +:mod:`pcapkit` must resolve, not the scripts' dependencies. """ @@ -72,7 +72,7 @@ def test_the_test_extra_carries_them_with_their_markers(self) -> None: self.assertEqual({name: test.get(name, 'absent') for name in SCRIPT_DEPENDENCIES}, { 'requests': None, 'psutil': "platform_python_implementation == 'CPython'", - 'lxml': "platform_python_implementation != 'GraalVM'", + 'lxml': "platform_python_implementation == 'CPython'", 'markdown-it-py': None, }) diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index f54d955d17..dfd7e2a2e5 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -73,13 +73,17 @@ GATING = ('required-checks', 'unittest-ordering') #: The unit tier's own flags, which every leg passes. TIER_FLAGS = ('--ignore=tests/integration', '--ignore-glob=*_runtime.py', '--ignore-glob=*_regression.py') -#: A ``python`` that writes each argument on a line of ``$ARGS_FILE``, and hands a -#: ``-c`` to the real one, which is how the steps ask tests/_tiers.py. util/cap_file_size.py's -#: command it execs, as the script does, with no real interpreter: under the coverage -#: leg one would be measured, be logged as a second ``coverage run --rcfile`` -#: controller, and exec before saving, which fails the #1387 guard (#1969). +#: A ``python`` that appends each argument on a line of ``$ARGS_FILE``, then an +#: :data:`END` line, and hands a ``-c`` to the real one, which is how the steps ask +#: tests/_tiers.py. util/cap_file_size.py's command it execs, as the script does, with +#: no real interpreter: under the coverage leg one would be measured, be logged as a +#: second ``coverage run --rcfile`` controller, and exec before saving, which fails the +#: #1387 guard (#1969). A PyPy leg runs pytest more than once (#2005), so each run is +#: recorded rather than the last. STUB = ('#!/bin/sh\ncase "$1" in -c) exec "$REAL_PYTHON" "$@";; util/cap_file_size.py) shift; exec "$@";; esac\n' - 'printf \'%s\\n\' "$@" > "$ARGS_FILE"\n') + 'printf \'%s\\n\' "$@" \'\' >> "$ARGS_FILE"\n') +#: The line :data:`STUB` writes after each run's arguments. +END = '' #: A directory holding sweeps, hand-picked round trips, and modules that only import #: the sweeps' harnesses' packet builders, for :class:`TestTheIgnoresAreTheLatestTier`. PROBE = 'tests/protocols/internet' @@ -117,7 +121,17 @@ def step_env(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB) -> 'dict[str def run_step(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB, script: 'Optional[str]' = None) -> 'list[str]': - """The arguments the step of job ``where`` hands ``python`` on ``leg``; ``script`` replaces its ``run:``.""" + """The arguments the step of job ``where`` hands ``python`` on ``leg``, which runs it once.""" + runs = run_step_each(leg, where, script) + if len(runs) != 1: + raise AssertionError(f'the step ran python {len(runs)} times on {leg}, not once') + return runs[0] + + +def run_step_each(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB, + script: 'Optional[str]' = None) -> 'list[list[str]]': + """The arguments of each run of ``python`` by the step of job ``where`` on ``leg``, in order; + ``script`` replaces its ``run:``.""" with tempfile.TemporaryDirectory() as temp: stub = pathlib.Path(temp, 'python') stub.write_text(STUB, encoding='utf-8') @@ -140,7 +154,40 @@ def run_step(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB, script: 'Opt cwd=ROOT, capture_output=True, text=True, check=False, timeout=120) if result.returncode: raise AssertionError(f'the step failed on {leg}: {result.stdout[-2000:]}{result.stderr[-2000:]}') - return [arg.replace(str(runner_temp), RUNNER_TEMP) for arg in record.read_text(encoding='utf-8').splitlines()] + runs, run = [], [] # type: list[list[str]], list[str] + for arg in record.read_text(encoding='utf-8').splitlines(): + if arg == END: + runs.append(run) + run = [] + else: + run.append(arg.replace(str(runner_temp), RUNNER_TEMP)) + return runs + + +def shards(leg: 'dict[str, Any]') -> 'list[str]': + """#2005: the directories the ``test`` step runs on their own on ``leg``, before the rest.""" + return step_env(leg).get('SHARDS', '').split() + + +def unsharded(leg: 'dict[str, Any]', runs: 'list[list[str]]') -> 'list[str]': + """The arguments each of the ``test`` step's runs on ``leg`` shares, once each run's own are taken off. + + #2005: a run per directory :func:`shards` names adds its JUnit file and the + directory, and the last run an ``--ignore`` of each. + + """ + named = shards(leg) + if len(runs) != len(named) + 1: + raise AssertionError(f'{len(runs)} runs on {leg} for the shards {named}') + rest = [f'--ignore={shard}' for shard in named] + common = runs[-1][:len(runs[-1]) - len(rest)] + if runs[-1][len(common):] != rest: + raise AssertionError(f'the last run on {leg} does not end by ignoring {named}: {runs[-1]}') + for shard, run in zip(named, runs): + own = [f'--junitxml={RUNNER_TEMP}/junit-{shard.replace("/", "-")}.xml', shard] + if run != common + own: + raise AssertionError(f'the {shard} run on {leg} is not the others\' arguments and {own}: {run}') + return common def pytest_args(argv: 'list[str]') -> 'list[str]': @@ -169,10 +216,10 @@ def test_the_coverage_command_is_recorded_without_the_real_interpreter(self) -> capture_output=True, text=True, check=False, timeout=60) self.assertEqual(result.returncode, 0, result.stderr) self.assertFalse(real.exists(), 'the real interpreter ran the capped coverage command') - self.assertEqual(record.read_text(encoding='utf-8').splitlines(), command) + self.assertEqual(record.read_text(encoding='utf-8').splitlines(), [*command, END]) -def unsharded(leg: 'dict[str, Any]', where: 'tuple[str, str]', argv: 'list[str]') -> 'list[str]': +def rest_unsharded(leg: 'dict[str, Any]', where: 'tuple[str, str]', argv: 'list[str]') -> 'list[str]': """``argv`` less what util/pytest_shards.py's last run adds on a sharded leg (#1963), which it must end with.""" shards = step_env(leg, where).get('SHARDS', '') if not shards: @@ -195,7 +242,10 @@ class TestTheStepSelectsByLeg(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.latest = live_outputs()['latest'] - cls.runs = {leg['python-version']: (leg, unsharded(leg, where, run_step(leg, where))) + # #2005: a PyPy leg of `test` runs pytest once a shard, each with the arguments the others share; + # #1963: GraalPy's `experimental` leg through util/pytest_shards.py, whose last run is the rest. + cls.runs = {leg['python-version']: (leg, unsharded(leg, run_step_each(leg, where)) if where == JOB + else rest_unsharded(leg, where, run_step_each(leg, where)[-1])) for where in (JOB, EXPERIMENTAL) for leg in legs(*where)} assert len(cls.runs) == len(legs(*JOB)) + len(legs(*EXPERIMENTAL)), 'a version runs in both jobs' @@ -246,7 +296,7 @@ def test_a_step_that_ignores_the_pins_alone_off_latest_is_caught(self) -> None: latest = _tiers.LATEST_TIER_SELECTOR.split('"', 1)[1] self.assertEqual(script.count(latest), 1) doctored = script.replace(latest, _tiers.PROJECT_PINS_SELECTOR.split('"', 1)[1]) - leg = next(leg for leg in legs(*JOB) if leg['python-version'] != self.latest) + leg = next(leg for leg in legs(*JOB) if leg['python-version'] != self.latest and not shards(leg)) self.assertNotEqual(ignored(run_step(leg, JOB, doctored)), list(_tiers.latest_tier_paths())) diff --git a/tests/protocols/misc/pcap/test_header_frame_unit.py b/tests/protocols/misc/pcap/test_header_frame_unit.py index b733134b7f..0c19d32dc8 100644 --- a/tests/protocols/misc/pcap/test_header_frame_unit.py +++ b/tests/protocols/misc/pcap/test_header_frame_unit.py @@ -307,7 +307,12 @@ def fromtimestamp(cls, *args): bad_time._nsec = False bad_time._fnum = 6 bad_time._file = io.BytesIO(b'') - with mock.patch('pcapkit.protocols.misc.pcap.frame.datetime.datetime', FakeDateTime), \ + # Faked through the module's name in frame.py, not through + # ``datetime.datetime`` itself: PyPy's datetime is pure Python and + # reads its own ``datetime`` global, so faking that fails the real call + # the fake hands on to (#1958). + fake_module = SimpleNamespace(**dict(vars(datetime), datetime=FakeDateTime)) + with mock.patch('pcapkit.protocols.misc.pcap.frame.datetime', fake_module), \ mock.patch.object(Frame, '_decode_next_layer', lambda self, info, proto=None, length=None, packet=None: info), \ mock.patch('pcapkit.protocols.misc.pcap.frame.warn') as warn: diff --git a/tests/protocols/misc/test_pcapng_unit.py b/tests/protocols/misc/test_pcapng_unit.py index 32ee92f4c0..55f5669ae7 100644 --- a/tests/protocols/misc/test_pcapng_unit.py +++ b/tests/protocols/misc/test_pcapng_unit.py @@ -570,7 +570,12 @@ def fromtimestamp(cls, *args): pcapng._ctx = types.SimpleNamespace(interfaces=[interface], section=types.SimpleNamespace(byteorder='little')) pcapng._type = BlockType.Enhanced_Packet_Block - with mock.patch('pcapkit.protocols.misc.pcapng.datetime.datetime', FakeDateTime), \ + # Faked through the module's name in pcapng.py, not through + # ``datetime.datetime`` itself: PyPy's datetime is pure Python and + # reads its own ``datetime`` global, so faking that fails the real call + # the fake hands on to (#1958). + fake_module = types.SimpleNamespace(**dict(vars(datetime), datetime=FakeDateTime)) + with mock.patch('pcapkit.protocols.misc.pcapng.datetime', fake_module), \ mock.patch('pcapkit.protocols.misc.pcapng.warn') as warn: timestamp, epoch = pcapng._read_timestamp(0, 1, interface_id=0) self.assertEqual(timestamp, real_datetime.fromtimestamp(0, datetime.timezone.utc)) diff --git a/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py b/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py index 202356f3b6..009040f0fb 100644 --- a/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py +++ b/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py @@ -17,8 +17,9 @@ #: GraalPy's :mod:`abc` is ``_py_abc``, which keeps its registry and caches as separate #: attributes, not in the C accelerator's ``_abc_impl`` these tests inspect (#1963). -NO_ABC_IMPL = sys.implementation.name == 'graalpy' -NO_ABC_IMPL_REASON = "GraalPy's abc is _py_abc, which has no _abc_impl (#1963)" +#: PyPy's built-in ``_abc`` keeps them on the class the same way (#1958). +NO_ABC_IMPL = sys.implementation.name in ('graalpy', 'pypy') +NO_ABC_IMPL_REASON = "GraalPy's abc is _py_abc and PyPy's _abc keeps _py_abc's attributes, so no _abc_impl" @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index 52511d1999..6ec1c19279 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -385,7 +385,8 @@ def test_the_capped_command_is_recorded_without_the_real_interpreter(self) -> No def leg_ignores(text: 'str', latest: 'str') -> 'list[str]': """The modules a ``test`` leg's ``Run unit tests`` step ``--ignore``\\ s, ``latest`` or not.""" script = run_script(job_section(text, 'test'), 'Run unit tests') - argv = stubbed_argv(script, {'ANNOTATION_LEVEL': 'error', 'COVERAGE': 'false', 'LATEST': latest}) + # #2005: a leg that runs pytest once, as a PyPy leg's own `SHARDS` would otherwise reach the stubbed step. + argv = stubbed_argv(script, {'ANNOTATION_LEVEL': 'error', 'COVERAGE': 'false', 'LATEST': latest, 'SHARDS': ''}) return [arg.split('=', 1)[1] for arg in argv if arg.startswith('--ignore=') and arg != f'--ignore=tests/{next(iter(_tiers.FIXTURE_TIER_DIRS))}'] diff --git a/tests/utilities/test_compat.py b/tests/utilities/test_compat.py index d5f1a9e69b..7ec21e9265 100644 --- a/tests/utilities/test_compat.py +++ b/tests/utilities/test_compat.py @@ -586,6 +586,8 @@ class OldFlag(enum.IntFlag): with self.assertRaises(ValueError): compat.show_flag_values(-1) + @unittest.skipIf(sys.implementation.name == 'pypy', + "PyPy's _thread has no _excepthook, and no tier runs a PyPy before 3.10 (#1958)") def test_python39_default_threading_excepthook_is_the_310_one(self) -> None: """The 3.8 and 3.9 branch lands on the object 3.10 and later name. diff --git a/tests/utilities/test_compat_aenum_nonetype_1958_unit.py b/tests/utilities/test_compat_aenum_nonetype_1958_unit.py new file mode 100644 index 0000000000..b608d720bb --- /dev/null +++ b/tests/utilities/test_compat_aenum_nonetype_1958_unit.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +"""#1958: :mod:`pcapkit` imports on PyPy 8.0, whose :mod:`builtins` binds ``NoneType``. + +aenum 3.1.17's ``_common`` binds a ``NoneType`` of its own only where the name +raises :exc:`NameError`, and aenum's ``__init__`` then star-imports ``_common``, +whose ``__all__`` lists the name. PyPy 8.0 binds it in :mod:`builtins`, so there +``_common`` binds none and ``import aenum`` failed with ``AttributeError: module +'aenum._common' has no attribute 'NoneType'``, and :mod:`pcapkit` with it. +:mod:`pcapkit.utilities.compat` binds the name in ``aenum._common`` and imports +aenum again. + +Each test runs a fresh interpreter, so that this module's own imports have not +loaded aenum already. :class:`TheAenumPremiseTests` binds the builtin on any +interpreter and shows aenum still fails without the fix, which tells when an +aenum release makes it unneeded. :class:`PyPyImportTests` runs on PyPy, where +the shim does. + +""" + +from __future__ import annotations + +import os +import subprocess # nosec: B404 +import sys +import textwrap +import unittest + +#: Test tier (#1958): a compat shim behind an implementation gate. +__tier__ = 'ALL' + +#: Repository root, first on the child's path, so it imports this tree. +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +#: Whether the shim in pcapkit/utilities/compat.py runs here. +ON_PYPY = sys.implementation.name == 'pypy' + + +def run_child(code: 'str') -> 'subprocess.CompletedProcess[str]': + """Run ``code`` in a fresh interpreter, with this tree first on its path.""" + env = dict(os.environ, PYTHONPATH=os.pathsep.join([REPO_ROOT] + [entry for entry in sys.path if entry]), + PYTHONSAFEPATH='1', PCAPKIT_DEVMODE='0') + return subprocess.run([sys.executable, '-c', textwrap.dedent(code)], # nosec: B603 + cwd=REPO_ROOT, env=env, capture_output=True, text=True, timeout=120, check=False) + + +class TheAenumPremiseTests(unittest.TestCase): + """aenum, imported on its own, still fails where :mod:`builtins` binds ``NoneType``.""" + + def test_aenum_alone_fails_on_the_builtin(self) -> 'None': + proc = run_child(''' + import builtins + builtins.NoneType = type(None) + try: + import aenum + except AttributeError as error: + print('FAILED', error) + else: + print('IMPORTED') + ''') + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout.split()[0], 'FAILED', + 'aenum imports with builtins.NoneType bound; the shim in ' + 'pcapkit/utilities/compat.py may no longer be needed') + self.assertIn("'NoneType'", proc.stdout) + + +@unittest.skipUnless(ON_PYPY, 'the shim runs on PyPy alone') +class PyPyImportTests(unittest.TestCase): + """On PyPy, :mod:`pcapkit` imports, and leaves :mod:`builtins` as PyPy has it.""" + + def test_pcapkit_imports_and_builtins_keep_none_type(self) -> 'None': + proc = run_child(''' + import builtins + import sys + print('BEFORE', 'NoneType' in vars(builtins)) + import pcapkit + import aenum + print('FILE', pcapkit.__file__) + print('AFTER', 'NoneType' in vars(builtins)) + print('AENUM', aenum.NoneType is type(None), sys.modules['aenum._common'].NoneType is type(None)) + ''') + self.assertEqual(proc.returncode, 0, proc.stderr) + lines = dict(line.split(' ', 1) for line in proc.stdout.splitlines() if ' ' in line) + self.assertTrue(lines['FILE'].startswith(REPO_ROOT), lines['FILE']) + self.assertEqual(lines['AFTER'], lines['BEFORE'], 'importing pcapkit changed builtins.NoneType') + self.assertEqual(lines['AENUM'], 'True True') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/utilities/test_exceptions_excepthook.py b/tests/utilities/test_exceptions_excepthook.py index d6eb25b599..f58659cb7d 100644 --- a/tests/utilities/test_exceptions_excepthook.py +++ b/tests/utilities/test_exceptions_excepthook.py @@ -269,6 +269,9 @@ def previous_hook(etype, value, tb): self.assertNotIn('IntError', proc.stderr.split('PREVIOUS_HOOK_RAN')[-1]) +@unittest.skipIf(sys.implementation.name == 'pypy', + "PyPy's sys.__excepthook__ under tracebacklimit = 0 prints the last exception without " + 'its chain, so it is no reference for the shape (#1958)') class ChainedExceptionTests(unittest.TestCase): """A chained exception must come out in the *shape* ``tracebacklimit = 0`` produced, not just "one line". diff --git a/util/python_versions.py b/util/python_versions.py index 98d66370d9..d88ee0277a 100644 --- a/util/python_versions.py +++ b/util/python_versions.py @@ -17,11 +17,15 @@ Both tiers. A KTLO leg carries ``ktlo: true``, and ``os`` where ``ubuntu-latest`` cannot provide it. ``test``, ``integration`` - The supported tier alone; ``test`` marks the ``latest`` leg ``coverage: true``. -``test-ktlo``, ``integration-ktlo``, ``pypcap-parity-ktlo`` - Each KTLO version as a leg carrying ``ktlo: true``, and ``os`` as above, for - unit-tests.yml's jobs of the same names, which nothing gating waits on (#1738). - No other unit-tests.yml output holds a KTLO version. + The supported tier alone, CPython's then PyPy's (#1958); ``test`` marks the + ``latest`` leg ``coverage: true``. +``test-ktlo``, ``integration-ktlo`` + Each KTLO version, CPython's then PyPy's, as a leg carrying ``ktlo: true``, + and ``os`` as above, for unit-tests.yml's jobs of the same names, which + nothing gating waits on (#1738). +``pypcap-parity-ktlo`` + The same, for CPython's KTLO versions below 3.12. No other unit-tests.yml + output holds a KTLO version. ``experimental`` Each experimental version as a leg carrying ``experimental: true``, for experimental.yml's ``experimental`` job, a workflow of its own so that nothing @@ -74,6 +78,10 @@ 'graalpy': re.compile(r'graalpy-(\d+)\.(\d+)')} #: The implementations every file lists; a GraalPy table is optional. MANDATORY = ('cpython', 'pypy') +#: The implementations unit-tests.yml's ``test`` and ``integration`` jobs and +#: their KTLO twins run, in this order: each runs the ``ALL`` test tier, and +#: ``latest`` the whole suite (#1958). +TESTED = ('cpython', 'pypy') TIERS = ('supported', 'ktlo', 'pre-release', 'experimental') #: The images a version may name as its ``runner``. RUNNER = re.compile(r'ubuntu-\d{2}\.04') @@ -209,8 +217,14 @@ def tiered(data: 'dict[str, Any]', impl: 'str' = 'cpython') -> 'dict[str, Any]': def supported_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': - """unit-tests.yml's ``integration``: the supported tier alone, its KTLO legs ``integration-ktlo``'s (#1738).""" - return {'python-version': tier(data, 'cpython', 'supported'), 'include': []} + """unit-tests.yml's ``integration``: the supported tier of each of :data:`TESTED`. + + Its KTLO legs are ``integration-ktlo``'s (#1738). PyPy's legs (#1958) run the + ``ALL`` tier, as every leg but ``latest`` does. + + """ + return {'python-version': [version for impl in TESTED for version in tier(data, impl, 'supported')], + 'include': []} def legs_matrix(found: 'list[dict[str, Any]]') -> 'dict[str, Any]': @@ -221,12 +235,14 @@ def legs_matrix(found: 'list[dict[str, Any]]') -> 'dict[str, Any]': def ktlo_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': """unit-tests.yml's ``test-ktlo`` and ``integration-ktlo``: the KTLO legs alone, out of the gating jobs (#1738). + Each of :data:`TESTED`'s, CPython's then PyPy's (#1958). + ``Required checks passed`` reads one ``needs..result`` per job, and ``unittest-ordering`` needs ``test``, so a KTLO leg that fails as a job -- a lost runner, or the job cap -- failed both from inside ``test``. """ - return legs_matrix(ktlo_legs(data)) + return legs_matrix([leg for impl in TESTED for leg in ktlo_legs(data, impl)]) def experimental_legs(data: 'dict[str, Any]') -> 'list[dict[str, Any]]': @@ -242,7 +258,7 @@ def experimental_legs(data: 'dict[str, Any]') -> 'list[dict[str, Any]]': def test_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': - """unit-tests.yml's ``test``: the supported tier, and the one coverage leg on ``latest``.""" + """unit-tests.yml's ``test``: ``integration``'s legs, and the one coverage leg on ``latest``.""" matrix = supported_matrix(data) matrix['include'].insert(0, {'python-version': data['latest'], 'coverage': True}) return matrix From f24c1b16eb54dbef67812c35682964e9bc47eada Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 21 Oct 2026 15:00:17 -0400 Subject: [PATCH 2/2] ci: run PyPy's shards through util/pytest_shards.py (#2005, #1963) - unit-tests.yml `test` / `Run unit tests`: the PyPy leg's inline shard loop becomes `python util/cap_file_size.py python util/pytest_shards.py "$SHARDS" -- python -m pytest "$@"`, #1963's script, on the leg's own interpreter (the #1969 pin requires the cap to exec `python`). Same runs, JUnit names and exit status; later shards also ignore earlier ones, and no run starts after SIGINT. - Pins: 1732's STUB runs the script on the real interpreter and `unsharded` accepts the earlier-shard ignores; 2005's test checks the invocation and each run's JUnit file. - workflows.rst: the PyPy paragraph and the Workflow Scripts row name the script; the `unit-tests.yml:NNNN` citations move 6 lines up (checked by content), here and in releasing.rst. --- .github/workflows/unit-tests.yml | 24 ++++++--------- docs/source/contributing/releasing.rst | 4 +-- docs/source/contributing/workflows.rst | 30 +++++++++++-------- tests/project/test_pypy_worker_memory_2005.py | 12 ++++++-- .../test_sweeps_on_latest_only_1732.py | 15 ++++++---- util/pytest_shards.py | 3 ++ 6 files changed, 49 insertions(+), 39 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 622821d145..12e26011ad 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -264,12 +264,13 @@ jobs: # at `-n 4` the `ALL` tier took each worker to 5.4-5.9 GB and them all # past 20 GB, measured on PyPy 8.0.1, where a 14 GB cap OOM-killed them # and a 3 GB `PYPY_GC_MAX` raised MemoryError instead. A smaller `-n` - # only hands each worker more tests. So a PyPy leg runs pytest once per - # directory `SHARDS` names, then once more over the rest, ignoring each - # of them, and every run starts its workers afresh: the largest, with - # tests/protocols, peaked at 2.6 GB a worker and 8.5 GB in all, and the - # tier took 383 s rather than 493 s. Each run but the last writes a - # JUnit file of its own, beside junit.xml; the step fails if any run did. + # only hands each worker more tests. So on a PyPy leg util/pytest_shards.py + # runs pytest once per directory `SHARDS` names, then once more over the + # rest, ignoring each of them, and every run starts its workers afresh: + # the largest, with tests/protocols, peaked at 2.6 GB a worker and 8.5 GB + # in all, and the tier took 383 s rather than 493 s. Each run but the + # last writes a JUnit file of its own, beside junit.xml; the step fails + # if any run did. env: COVERAGE: ${{ matrix.coverage }} LATEST: ${{ matrix.python-version == needs.python-versions.outputs.latest }} @@ -297,15 +298,8 @@ jobs: COVERAGE_DEBUG=pid,process COVERAGE_DEBUG_FILE="$PWD/coverage-processes.log" \ python util/cap_file_size.py python -m coverage run --rcfile=.github/coverage.toml -m pytest "$@" elif [ -n "$SHARDS" ]; then - status=0 - rest=() - for shard in $SHARDS; do - python util/cap_file_size.py python -m pytest "$@" \ - --junitxml="$RUNNER_TEMP/junit-${shard//\//-}.xml" "$shard" || status=$? - rest+=(--ignore="$shard") - done - python util/cap_file_size.py python -m pytest "$@" "${rest[@]}" || status=$? - exit "$status" + # #2005: #1963's script, on the leg's interpreter, under the one cap. + python util/cap_file_size.py python util/pytest_shards.py "$SHARDS" -- python -m pytest "$@" else python util/cap_file_size.py python -m pytest "$@" fi diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 8f6db0e1fc..b26503ecc6 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1804-1879``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1798-1873``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1647-1673``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1641-1667``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 1e0935e299..ceb97a6e39 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -376,7 +376,7 @@ the same commit several times over. It also skips ``changes``, Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1635-1641``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1629-1635``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -689,7 +689,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1970`` + - ``unit-tests.yml:1964`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -711,22 +711,22 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1970``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:1964``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1882`` mentions the string without defining it. +``unit-tests.yml:1876`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1881-1891``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1875-1885``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, and ``changes`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1969-2025``). Rulesets match check names literally, with no +(``unit-tests.yml:1963-2019``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where @@ -735,7 +735,7 @@ It accepts the gating jobs as ``skipped`` only on a docs-only pull request, wher ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1972``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1966``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -759,7 +759,7 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:400-417``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:394-411``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -797,9 +797,10 @@ with the image's Rust and ``libssl-dev``, and none of ``lxml``, so the ``test`` extra carries ``lxml`` on CPython only. On PyPy an xdist worker's RSS grows with every test it runs, rather than in a few -heavy modules, so a PyPy leg's ``Run unit tests`` runs pytest once per directory its -``SHARDS`` names, ``tests/protocols`` and then ``tests/corekit``, and once more over -the rest, ignoring each (`#2005 `__). +heavy modules, so a PyPy leg's ``Run unit tests`` hands its pytest command to +:file:`util/pytest_shards.py`, as GraalPy's leg does (see below), which runs it once per +directory its ``SHARDS`` names, ``tests/protocols`` and then ``tests/corekit``, and once +more over the rest, ignoring each (`#2005 `__). Each run starts its workers afresh. Measured on PyPy 8.0.1 at ``-n 4``, one run over the whole ``ALL`` tier took each worker to 5.4-5.9 GB and the workers together past 20 GB, beyond the runner's 16 GB; split, the largest run peaked at 2.6 GB a @@ -1201,14 +1202,17 @@ does; each has tests under :file:`tests/project/` that run it. - ``Report coverage`` in ``unit-tests.yml``'s ``test``, on the coverage leg. * - :file:`util/pytest_shards.py` - ``Run unit tests`` in ``experimental.yml``'s ``experimental``, on GraalPy's leg - (`#1963 `__). + (`#1963 `__), and in + ``unit-tests.yml``'s ``test`` and ``test-ktlo``, on PyPy's legs (`#2005 + `__). The summaries run on the image's ``/usr/bin/python3`` rather than the leg's interpreter, which may have failed to set up or be too broken to run anything, so they need only the standard library, as ``classify_changes.py`` does on the runner's ``python3``. :file:`util/pytest_shards.py` runs on ``/usr/bin/python3`` too, standard library only, so on GraalPy's leg the leg's interpreter runs pytest -alone. Both ``changes`` jobs check out on every event, since the script is in the +alone; on PyPy's legs the leg's interpreter runs the script as well, under the +file-size cap every test command of ``unit-tests.yml`` takes. Both ``changes`` jobs check out on every event, since the script is in the tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of the KTLO summary's shell until its shell moves to scripts too; a project pin runs that copy and the script on the same steps and requires the same output. diff --git a/tests/project/test_pypy_worker_memory_2005.py b/tests/project/test_pypy_worker_memory_2005.py index 2ad7b7dfcf..6463107edf 100644 --- a/tests/project/test_pypy_worker_memory_2005.py +++ b/tests/project/test_pypy_worker_memory_2005.py @@ -91,7 +91,7 @@ def test_a_shard_that_fails_fails_the_step_after_the_rest_ran(self) -> None: with tempfile.TemporaryDirectory() as temp: stub = pathlib.Path(temp, 'python') # The first pytest run fails as pytest does on a failed test; `-c` asks the real interpreter. - stub.write_text('#!/bin/sh\ncase "$1" in -c) exec "$REAL_PYTHON" "$@";; ' + stub.write_text('#!/bin/sh\ncase "$1" in -c|util/pytest_shards.py) exec "$REAL_PYTHON" "$@";; ' 'util/cap_file_size.py) shift; exec "$@";; esac\n' 'echo run >> "$RUNS"\n' '[ "$(wc -l < "$RUNS")" -eq 1 ] && exit 1\nexit 0\n', encoding='utf-8') @@ -112,9 +112,15 @@ class TestTheTimingsAreUploaded(unittest.TestCase): def test_the_upload_takes_each_runs_junit_file(self) -> None: upload = next(entry for entry in job(*JOB)['steps'] if entry.get('name') == 'Upload test timings') self.assertEqual(upload['with']['path'], '${{ runner.temp }}/junit*.xml') - for shard in SHARDS: + runs = run_step_each(pypy_leg()) + for shard, run in zip(SHARDS, runs): with self.subTest(shard=shard): - self.assertIn(f'--junitxml="$RUNNER_TEMP/junit-${{shard//\\//-}}.xml" "$shard"', step()['run']) + self.assertIn(f'--junitxml={RUNNER_TEMP}/junit-{shard.replace("/", "-")}.xml', run) + + def test_the_shards_run_through_the_graalpy_script(self) -> None: + """#2005 reuses #1963's util/pytest_shards.py, on the leg's interpreter, under the one file-size cap.""" + self.assertIn('python util/cap_file_size.py python util/pytest_shards.py "$SHARDS" -- ' + 'python -m pytest "$@"', step()['run']) class TestTheDocsSayIt(unittest.TestCase): diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index dfd7e2a2e5..a783830eaf 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -79,8 +79,10 @@ #: no real interpreter: under the coverage leg one would be measured, be logged as a #: second ``coverage run --rcfile`` controller, and exec before saving, which fails the #: #1387 guard (#1969). A PyPy leg runs pytest more than once (#2005), so each run is -#: recorded rather than the last. -STUB = ('#!/bin/sh\ncase "$1" in -c) exec "$REAL_PYTHON" "$@";; util/cap_file_size.py) shift; exec "$@";; esac\n' +#: recorded rather than the last, and util/pytest_shards.py, which starts them there, +#: runs on the real one too. +STUB = ('#!/bin/sh\ncase "$1" in -c|util/pytest_shards.py) exec "$REAL_PYTHON" "$@";; ' + 'util/cap_file_size.py) shift; exec "$@";; esac\n' 'printf \'%s\\n\' "$@" \'\' >> "$ARGS_FILE"\n') #: The line :data:`STUB` writes after each run's arguments. END = '' @@ -172,8 +174,9 @@ def shards(leg: 'dict[str, Any]') -> 'list[str]': def unsharded(leg: 'dict[str, Any]', runs: 'list[list[str]]') -> 'list[str]': """The arguments each of the ``test`` step's runs on ``leg`` shares, once each run's own are taken off. - #2005: a run per directory :func:`shards` names adds its JUnit file and the - directory, and the last run an ``--ignore`` of each. + #2005: util/pytest_shards.py's run per directory :func:`shards` names adds its JUnit + file, an ``--ignore`` of each earlier shard, and the directory, and the last run an + ``--ignore`` of each. """ named = shards(leg) @@ -183,8 +186,8 @@ def unsharded(leg: 'dict[str, Any]', runs: 'list[list[str]]') -> 'list[str]': common = runs[-1][:len(runs[-1]) - len(rest)] if runs[-1][len(common):] != rest: raise AssertionError(f'the last run on {leg} does not end by ignoring {named}: {runs[-1]}') - for shard, run in zip(named, runs): - own = [f'--junitxml={RUNNER_TEMP}/junit-{shard.replace("/", "-")}.xml', shard] + for index, (shard, run) in enumerate(zip(named, runs)): + own = [f'--junitxml={RUNNER_TEMP}/junit-{shard.replace("/", "-")}.xml', *rest[:index], shard] if run != common + own: raise AssertionError(f'the {shard} run on {leg} is not the others\' arguments and {own}: {run}') return common diff --git a/util/pytest_shards.py b/util/pytest_shards.py index 72b54b3b57..8204d4f186 100644 --- a/util/pytest_shards.py +++ b/util/pytest_shards.py @@ -9,6 +9,9 @@ /usr/bin/python3 util/pytest_shards.py "$SHARDS" -- python -m pytest ... +unit-tests.yml's ``Run unit tests`` runs it the same way on PyPy's legs, whose +xdist workers' RSS grows with every test as well (#2005). + ``SHARDS`` holds the shards, separated by whitespace, and each is a directory or a glob of the entries of one directory, such as ``tests/corekit/test_i*``. The command runs once per shard, then once more over the rest, and each run