From 775aa5a29d5f0b020c4480dcbd1dfec66201b5b1 Mon Sep 17 00:00:00 2001 From: Alexey Shalaev <75322386+AlexeyShalaev@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:37:16 +0400 Subject: [PATCH] perf: import clientwright and capabilities_matrix() without loading asyncio core.policy.concurrency imported asyncio at module level and sits on the import path of the package root, so every `import clientwright` paid for asyncio, ssl, socket and subprocess. AsyncOriginLimiter now imports asyncio where it creates a semaphore, inside a running loop. capabilities_matrix() loaded the httpx and httpx2 records through _httpx_shared, which imports asyncio, ssl and both engines. The family's capability record now lives in _httpx_capabilities, which needs only the core's capability and model types. Closes #62 --- .agents/project.md | 2 +- clientwright/adapters/_httpx_capabilities.py | 88 ++++++++++++++++++++ clientwright/adapters/_httpx_shared.py | 83 ++---------------- clientwright/adapters/httpx/capabilities.py | 2 +- clientwright/adapters/httpx2/capabilities.py | 2 +- clientwright/core/policy/concurrency.py | 12 ++- docs/advanced/architecture.md | 4 +- docs/advanced/writing-an-adapter.md | 5 +- docs/learn/install.md | 4 +- pyproject.toml | 6 +- tests/unit/adapters/test_bare_install.py | 28 +++++++ 11 files changed, 149 insertions(+), 87 deletions(-) create mode 100644 clientwright/adapters/_httpx_capabilities.py diff --git a/.agents/project.md b/.agents/project.md index 28eb022..219a35e 100644 --- a/.agents/project.md +++ b/.agents/project.md @@ -32,7 +32,7 @@ - `CHANGELOG.md` is generated by release-please: never edit it by hand, and never add an `## [Unreleased]` section. `docs/changelog.md` is a copy of it made for the docs build and ignored by git. - The version lives in `clientwright/__version__.py`, on the line marked `# x-release-please-version`, and in `.release-please-manifest.json`; release-please bumps both. Never change them by hand. - `docs/agents.md` is part of the public API: it changes in the same pull request as the API (CONTRIBUTING.md, "The agents page"). -- `clientwright/core/` never imports `clientwright/adapters/` or any third-party package, and the adapters never import each other; `clientwright/adapters/_httpx_shared.py` is the one module httpx and httpx2 share. The import-linter contracts in `pyproject.toml` enforce this in `make check` and CI. +- `clientwright/core/` never imports `clientwright/adapters/` or any third-party package, and the adapters never import each other; `clientwright/adapters/_httpx_shared.py` is the one module httpx and httpx2 share, with their capability record split out into `clientwright/adapters/_httpx_capabilities.py` so that `capabilities_matrix()` stays light. The import-linter contracts in `pyproject.toml` enforce this in `make check` and CI. - Files from the engineering-assets hub (`AGENTS.md` explains how it works) include the copy-page files (`docs/assets/javascripts/copy-page.js`, `docs/assets/stylesheets/copy-page.css`, `overrides/main.html`, `scripts/emit_markdown.py`), `.github/workflows/docs.yml`, `.github/workflows/release-please.yml`, `.github/dependabot.yml`, `.editorconfig` and `CODE_OF_CONDUCT.md`. Change them in the hub; a change here makes the file this repository's own. - Never publish to PyPI or create a release by hand: release-please creates the release, and `publish.yml` publishes it. diff --git a/clientwright/adapters/_httpx_capabilities.py b/clientwright/adapters/_httpx_capabilities.py new file mode 100644 index 0000000..f5ad0ad --- /dev/null +++ b/clientwright/adapters/_httpx_capabilities.py @@ -0,0 +1,88 @@ +"""Capability record of the httpx family: httpx and its successor httpx2. + +Kept apart from ``_httpx_shared`` so that ``capabilities_matrix()`` stays light: +the capability modules import this one, which needs only the core's capability +and model types, not the engines, asyncio or ssl that ``_httpx_shared`` loads. +""" + +from __future__ import annotations + +from ..core.capabilities import ( + AdapterCapabilities, + Capability, + DurationBoundary, + SeamGranularity, + Support, +) +from ..core.model import FailureKind + + +def capabilities_for(adapter: str) -> AdapterCapabilities: + """The family capability record; only the adapter name differs.""" + return AdapterCapabilities( + adapter=adapter, + seam="transport", + granularity=SeamGranularity.HOP, + boundary=DurationBoundary.HEADERS, + support={ + Capability.TIMEOUT_TOTAL: Support.EMULATED, + Capability.TIMEOUT_ATTEMPT: Support.EMULATED, + Capability.TIMEOUT_CONNECT: Support.NATIVE, + Capability.TIMEOUT_READ: Support.NATIVE, + Capability.TIMEOUT_WRITE: Support.NATIVE, + Capability.TIMEOUT_POOL: Support.NATIVE, + Capability.DEADLINE_HARD: Support.EMULATED, + Capability.DEADLINE_COVERS_BODY: Support.EMULATED, + Capability.POOL_LIMIT_TOTAL: Support.NATIVE, + Capability.POOL_LIMIT_PER_HOST: Support.EMULATED, + Capability.KEEPALIVE: Support.NATIVE, + Capability.POOL_METRICS: Support.ABSENT, + Capability.CONN_METRICS: Support.ABSENT, + Capability.REDIRECTS_OWNABLE: Support.NATIVE, + Capability.NATIVE_RETRY_DISABLEABLE: Support.NATIVE, + Capability.PER_CALL_OPTIONS: Support.NATIVE, + Capability.RETROFIT: Support.ABSENT, + Capability.EXACT_NATIVE_TYPE: Support.NATIVE, + Capability.BALANCER: Support.ABSENT, + Capability.HTTP2: Support.NATIVE, + Capability.HTTP3: Support.ABSENT, + Capability.PROXY: Support.NATIVE, + }, + emits=frozenset( + { + FailureKind.CONNECT_TIMEOUT, + FailureKind.READ_TIMEOUT, + FailureKind.WRITE_TIMEOUT, + FailureKind.POOL_TIMEOUT, + FailureKind.ATTEMPT_TIMEOUT, + FailureKind.TOTAL_TIMEOUT, + FailureKind.CONNECT_ERROR, + FailureKind.TLS_ERROR, + FailureKind.PROTOCOL_ERROR, + FailureKind.DISCONNECTED, + FailureKind.BODY_ERROR, + FailureKind.STATUS, + FailureKind.CANCELLED, + FailureKind.CIRCUIT_OPEN, + FailureKind.UNKNOWN, + } + ), + collapses={FailureKind.DNS_ERROR: FailureKind.CONNECT_ERROR}, + notes={ + "deadline_hard": "Hard cancellation on the async client only; the sync client clamps phases (soft).", + "attempt_timeout": ( + "Emitted by the async client only; the sync client drops the attempt ceiling, so a stall arrives as " + "read_timeout." + ), + "pool_limit_per_host": "Emulated as a per-origin in-flight semaphore; limits requests, not connections.", + "dns_error": f"{adapter} wraps DNS failures into ConnectError; they surface as connect_error.", + "proxy_from_env": "Environment proxies are parsed into mounts; NO_PROXY entries match hosts literally.", + "deadline_covers_body": ( + "The response body is inside the total: the async client bounds every chunk by the remaining budget, " + "the sync client refuses the next chunk once the budget is gone (soft)." + ), + }, + ) + + +__all__ = ["capabilities_for"] diff --git a/clientwright/adapters/_httpx_shared.py b/clientwright/adapters/_httpx_shared.py index 874cc66..13bcd78 100644 --- a/clientwright/adapters/_httpx_shared.py +++ b/clientwright/adapters/_httpx_shared.py @@ -4,8 +4,10 @@ stays importable without either extra). Each adapter's modules hand their SDK namespace in and bind the SDK base classes (byte streams, transports, error families) to the logic mixins defined here. This is the ONE sanctioned channel -for code sharing between the paired adapters; the import-linter independence -contract still forbids importing a sibling adapter directly. +for code sharing between the paired adapters (the family's capability record is +split out into ``_httpx_capabilities``, which ``capabilities_matrix()`` loads +without this module); the import-linter independence contract still forbids +importing a sibling adapter directly. Public names in both adapters are IDENTICAL on purpose: migrating a service is one extra and one import path. @@ -18,13 +20,7 @@ from collections.abc import Callable, Mapping, MutableMapping from typing import Any -from ..core.capabilities import ( - AdapterCapabilities, - Capability, - DurationBoundary, - SeamGranularity, - Support, -) +from ..core.capabilities import AdapterCapabilities, Capability from ..core.config import ClientConfig, RedirectMode, is_set, resolve from ..core.contracts.adapter import AdapterDeps from ..core.engine.aio import AsyncAttemptEngine @@ -71,74 +67,6 @@ } -def capabilities_for(adapter: str) -> AdapterCapabilities: - """The family capability record; only the adapter name differs.""" - return AdapterCapabilities( - adapter=adapter, - seam="transport", - granularity=SeamGranularity.HOP, - boundary=DurationBoundary.HEADERS, - support={ - Capability.TIMEOUT_TOTAL: Support.EMULATED, - Capability.TIMEOUT_ATTEMPT: Support.EMULATED, - Capability.TIMEOUT_CONNECT: Support.NATIVE, - Capability.TIMEOUT_READ: Support.NATIVE, - Capability.TIMEOUT_WRITE: Support.NATIVE, - Capability.TIMEOUT_POOL: Support.NATIVE, - Capability.DEADLINE_HARD: Support.EMULATED, - Capability.DEADLINE_COVERS_BODY: Support.EMULATED, - Capability.POOL_LIMIT_TOTAL: Support.NATIVE, - Capability.POOL_LIMIT_PER_HOST: Support.EMULATED, - Capability.KEEPALIVE: Support.NATIVE, - Capability.POOL_METRICS: Support.ABSENT, - Capability.CONN_METRICS: Support.ABSENT, - Capability.REDIRECTS_OWNABLE: Support.NATIVE, - Capability.NATIVE_RETRY_DISABLEABLE: Support.NATIVE, - Capability.PER_CALL_OPTIONS: Support.NATIVE, - Capability.RETROFIT: Support.ABSENT, - Capability.EXACT_NATIVE_TYPE: Support.NATIVE, - Capability.BALANCER: Support.ABSENT, - Capability.HTTP2: Support.NATIVE, - Capability.HTTP3: Support.ABSENT, - Capability.PROXY: Support.NATIVE, - }, - emits=frozenset( - { - FailureKind.CONNECT_TIMEOUT, - FailureKind.READ_TIMEOUT, - FailureKind.WRITE_TIMEOUT, - FailureKind.POOL_TIMEOUT, - FailureKind.ATTEMPT_TIMEOUT, - FailureKind.TOTAL_TIMEOUT, - FailureKind.CONNECT_ERROR, - FailureKind.TLS_ERROR, - FailureKind.PROTOCOL_ERROR, - FailureKind.DISCONNECTED, - FailureKind.BODY_ERROR, - FailureKind.STATUS, - FailureKind.CANCELLED, - FailureKind.CIRCUIT_OPEN, - FailureKind.UNKNOWN, - } - ), - collapses={FailureKind.DNS_ERROR: FailureKind.CONNECT_ERROR}, - notes={ - "deadline_hard": "Hard cancellation on the async client only; the sync client clamps phases (soft).", - "attempt_timeout": ( - "Emitted by the async client only; the sync client drops the attempt ceiling, so a stall arrives as " - "read_timeout." - ), - "pool_limit_per_host": "Emulated as a per-origin in-flight semaphore; limits requests, not connections.", - "dns_error": f"{adapter} wraps DNS failures into ConnectError; they surface as connect_error.", - "proxy_from_env": "Environment proxies are parsed into mounts; NO_PROXY entries match hosts literally.", - "deadline_covers_body": ( - "The response body is inside the total: the async client bounds every chunk by the remaining budget, " - "the sync client refuses the next chunk once the budget is gone (soft)." - ), - }, - ) - - def has_ssl_cause(exc: BaseException) -> bool: seen: set[int] = set() current: BaseException | None = exc @@ -881,7 +809,6 @@ def make_raw(proxy: str | None) -> Any: "SyncProxyRouterMixin", "SyncTimedStreamMixin", "TimedStreamCore", - "capabilities_for", "classify_family_error", "host_bypasses_proxy", "make_error_translator", diff --git a/clientwright/adapters/httpx/capabilities.py b/clientwright/adapters/httpx/capabilities.py index 9aac2a4..c6ef5d2 100644 --- a/clientwright/adapters/httpx/capabilities.py +++ b/clientwright/adapters/httpx/capabilities.py @@ -2,7 +2,7 @@ from __future__ import annotations -from .._httpx_shared import capabilities_for +from .._httpx_capabilities import capabilities_for CAPABILITIES = capabilities_for("httpx") diff --git a/clientwright/adapters/httpx2/capabilities.py b/clientwright/adapters/httpx2/capabilities.py index 856f4d7..e9f8cec 100644 --- a/clientwright/adapters/httpx2/capabilities.py +++ b/clientwright/adapters/httpx2/capabilities.py @@ -2,7 +2,7 @@ from __future__ import annotations -from .._httpx_shared import capabilities_for +from .._httpx_capabilities import capabilities_for CAPABILITIES = capabilities_for("httpx2") diff --git a/clientwright/core/policy/concurrency.py b/clientwright/core/policy/concurrency.py index 0316a2d..79477d1 100644 --- a/clientwright/core/policy/concurrency.py +++ b/clientwright/core/policy/concurrency.py @@ -8,14 +8,22 @@ ``asyncio.run`` calls, test suites), and an ``asyncio.Semaphore`` binds to the loop of its first contended acquire. Per-loop state also dies with its loop, so slots consumed in a dead loop cannot shrink the limit forever. + +``asyncio`` is imported where a semaphore is first needed, not at module level: +this module is on the path of ``import clientwright``, and asyncio (with ssl, +socket and subprocess behind it) is the largest part of that import. Code that +only builds configuration, or only sync clients, never pays for it. """ from __future__ import annotations -import asyncio import threading import weakref from types import TracebackType +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import asyncio class AsyncOriginLimiter: @@ -29,6 +37,8 @@ def __init__(self, limit: int) -> None: self._lock = threading.Lock() def _semaphore(self, origin: str) -> asyncio.Semaphore: + import asyncio # noqa: PLC0415 - deferred off the `import clientwright` path, see the module docstring + loop = asyncio.get_running_loop() with self._lock: per_origin = self._per_loop.get(loop) diff --git a/docs/advanced/architecture.md b/docs/advanced/architecture.md index 05baf8d..9edd3a4 100644 --- a/docs/advanced/architecture.md +++ b/docs/advanced/architecture.md @@ -26,7 +26,9 @@ Three import-linter contracts hold the shape: the core never imports adapters, the core imports nothing outside the stdlib, and adapters never import each other. (The httpx and httpx2 adapters share a private `_httpx_shared` module that is deliberately outside the independence contract — it is the *family's* core, -SDK-agnostic by construction.) +SDK-agnostic by construction. Their capability record sits next to it in +`_httpx_capabilities`, so `capabilities_matrix()` loads neither the engines nor +asyncio.) ## One call through the machine diff --git a/docs/advanced/writing-an-adapter.md b/docs/advanced/writing-an-adapter.md index 831c4a2..62ae5c5 100644 --- a/docs/advanced/writing-an-adapter.md +++ b/docs/advanced/writing-an-adapter.md @@ -64,7 +64,10 @@ CAPABILITIES = AdapterCapabilities( ``` This module must import **without the SDK installed** — it is what -`capabilities_matrix()` shows to users deciding whether to adopt you. Understate +`capabilities_matrix()` shows to users deciding whether to adopt you. Keep it +light, too: import only `clientwright.core.capabilities` and +`clientwright.core.model`, not your transport or engine code, because the matrix +imports every registered adapter's record. Understate rather than overstate: `dropped` with a reason beats a knob that silently does nothing. `emits` also covers what the engine itself surfaces through your seam: `total_timeout`, `circuit_open`, `cancelled` and, on an async adapter, diff --git a/docs/learn/install.md b/docs/learn/install.md index db98bcb..bc17b35 100644 --- a/docs/learn/install.md +++ b/docs/learn/install.md @@ -40,7 +40,9 @@ pip install "clientwright[httpx,observability,deadline]" that is what lets the test suite and CI verify the core in complete isolation. Adapters load lazily: nothing touches `httpx` until you call `build("httpx", ...)`, and a missing extra fails with an install hint instead of - a bare `ImportError`. + a bare `ImportError`. Neither `import clientwright` nor `capabilities_matrix()` + loads `asyncio`, so a job that only builds `ClientConfig` and reads capabilities + starts quickly; asyncio loads when an async client first needs it. ## Version pinning diff --git a/pyproject.toml b/pyproject.toml index d9a32fe..dbff45b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,8 +185,10 @@ forbidden_modules = [ "opentelemetry", "structlog", "sentry_sdk", "servicewright", ] -# _httpx_shared is deliberately OUTSIDE this list: it is the one sanctioned -# sharing channel of the httpx family (httpx and httpx2 both import it). +# _httpx_shared and _httpx_capabilities are deliberately OUTSIDE this list: they +# are the one sanctioned sharing channel of the httpx family (httpx and httpx2 +# both import them). The capability record lives in its own module so that +# capabilities_matrix() does not load the engines and asyncio. [[tool.importlinter.contracts]] name = "Adapters never depend on each other" type = "independence" diff --git a/tests/unit/adapters/test_bare_install.py b/tests/unit/adapters/test_bare_install.py index 68e59c2..d1027f1 100644 --- a/tests/unit/adapters/test_bare_install.py +++ b/tests/unit/adapters/test_bare_install.py @@ -178,3 +178,31 @@ def test__contrib_servicewright_without_the_extra__fails_with_the_install_hint() """ ) assert "clientwright[servicewright]" in output + + +def test__import_clientwright__does_not_import_asyncio() -> None: + output = run_probe( + """ + import sys + + import clientwright + + print("asyncio loaded" if "asyncio" in sys.modules else "no asyncio") + """ + ) + assert output == "no asyncio" + + +def test__config_and_capabilities_matrix__do_not_import_asyncio() -> None: + output = run_probe( + """ + import sys + + import clientwright + + clientwright.ClientConfig(service_name="probe", base_url="https://example.com") + assert len(clientwright.capabilities_matrix()) == 5 + print("asyncio loaded" if "asyncio" in sys.modules else "no asyncio") + """ + ) + assert output == "no asyncio"