From bcb203b44aa387cf7fb863a0424b26445fb18226 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 22 Sep 2026 20:02:31 -0700 Subject: [PATCH 1/7] removed unused dependencies --- .../porting-to-canyonos/validation/runtime.py | 13 ++----------- .../llm_proxy/providers/__init__.py | 13 ++++++++++--- packages/core/canyonos_core/stub_generator.py | 16 ++-------------- packages/core/tests/test_stub_generator.py | 18 ++---------------- 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/.claude/skills/porting-to-canyonos/validation/runtime.py b/.claude/skills/porting-to-canyonos/validation/runtime.py index 0be85da8..2c6199c5 100644 --- a/.claude/skills/porting-to-canyonos/validation/runtime.py +++ b/.claude/skills/porting-to-canyonos/validation/runtime.py @@ -48,17 +48,8 @@ def _base_requirements(): - agent = [ - "grpcio", - "grpcio-tools", - "redis", - "pyyaml", - "psutil", - "ipdb", - "ipython", - "boto3", - ] - workflow = [*agent, "flask", "sqlalchemy", "psycopg[binary]"] + agent = ["grpcio", "protobuf", "redis", "flask", "requests"] + workflow = [*agent] try: from canyonos_core import stub_generator except Exception: # noqa: BLE001 - a broken install must not crash validation diff --git a/packages/core/canyonos_core/llm_proxy/providers/__init__.py b/packages/core/canyonos_core/llm_proxy/providers/__init__.py index 1ca3bd19..a32e19f6 100644 --- a/packages/core/canyonos_core/llm_proxy/providers/__init__.py +++ b/packages/core/canyonos_core/llm_proxy/providers/__init__.py @@ -1,14 +1,21 @@ from __future__ import annotations from canyonos_core.llm_proxy.providers.anthropic import AnthropicProvider -from canyonos_core.llm_proxy.providers.bedrock import BedrockProvider from canyonos_core.llm_proxy.providers.openai import OpenAIProvider +# boto3 ships only in images whose agent declares it, so Bedrock is registered only when present. +try: + from canyonos_core.llm_proxy.providers.bedrock import BedrockProvider +except ImportError: + BedrockProvider = None + def build_registry(cfg): """Map the URL prefix -> provider instance.""" - return { + registry = { "openai": OpenAIProvider(cfg), "anthropic": AnthropicProvider(cfg), - "bedrock": BedrockProvider(cfg), } + if BedrockProvider is not None: + registry["bedrock"] = BedrockProvider(cfg) + return registry diff --git a/packages/core/canyonos_core/stub_generator.py b/packages/core/canyonos_core/stub_generator.py index 771d5012..ca8e7c2c 100644 --- a/packages/core/canyonos_core/stub_generator.py +++ b/packages/core/canyonos_core/stub_generator.py @@ -18,23 +18,11 @@ from packaging.requirements import InvalidRequirement, Requirement from packaging.version import Version -# Packages every agent container needs regardless of its specific business logic. -# -# protobuf and grpcio-tools move together: grpcio-tools carries the only upper -# bound on protobuf here (1.65.5 capped it below 6.0), and a runtime older than -# the gencode of any *_pb2.py in the image refuses to load. Transitively -# installed packages ship gencode 6.x -- googleapis-common-protos, pulled in by -# the OTLP gRPC exporter, is one -- so a 5.x runtime crashed on import with -# "gencode 6.33.5 runtime 5.29.6". Neither uv nor pip can reject that pairing, -# because the constraint lives in the generated module, not in any metadata. +# Packages every agent container needs; protobuf must be at least the gencode version of any *_pb2.py in the image. BASE_AGENT_REQUIREMENTS = [ "grpcio==1.83.1", - "grpcio-tools==1.76.0", "protobuf==6.33.5", "redis==8.1.0", - "pyyaml==6.0.3", - "psutil==7.2.2", - "boto3==1.43.91", "flask==3.1.3", "requests==2.34.2", ] @@ -45,7 +33,7 @@ # Packages the image's own code is built against, so an app cannot be left to # pick them alone. -_FORCED_FROM_BASE = ("protobuf", "grpcio", "grpcio-tools", "requests", "boto3") +_FORCED_FROM_BASE = ("protobuf", "grpcio", "requests") PLATFORM_PINS = [ pin for pin in BASE_AGENT_REQUIREMENTS if pin.split("==")[0] in _FORCED_FROM_BASE ] diff --git a/packages/core/tests/test_stub_generator.py b/packages/core/tests/test_stub_generator.py index b9f2bc48..81fc2e03 100644 --- a/packages/core/tests/test_stub_generator.py +++ b/packages/core/tests/test_stub_generator.py @@ -53,12 +53,8 @@ def test_base_only_when_requirements_omitted(self): requirements, [ "grpcio==1.83.1", - "grpcio-tools==1.76.0", "protobuf==6.33.5", "redis==8.1.0", - "pyyaml==6.0.3", - "psutil==7.2.2", - "boto3==1.43.91", "flask==3.1.3", "requests==2.34.2", ], @@ -533,18 +529,8 @@ def test_pins_come_from_the_base_requirements(self): with self.subTest(pin=pin): self.assertIn(pin, BASE_AGENT_REQUIREMENTS) - def test_grpcio_tools_is_forced_wherever_protobuf_is(self): - # protoc stamps its own generation into the *_pb2.py it writes. - forced = {pin.split("==")[0] for pin in PLATFORM_PINS} - if "protobuf" in forced: - self.assertIn("grpcio-tools", forced) - - def test_the_forced_protobuf_satisfies_the_grpcio_tools_bound(self): - # grpcio-tools carries the only upper bound on protobuf in the base set, - # so the two cannot be bumped independently: 1.65.5 required - # protobuf<6.0, which held the runtime below the gencode 6.x that - # transitively installed *_pb2.py modules are built with. 1.76.0 - # requires >=6.31.1. + def test_the_forced_protobuf_loads_host_compiled_gencode(self): + # The host's grpcio-tools 1.76.0 emits *_pb2.py that need protobuf>=6.31.1 at runtime. pins = {pin.split("==")[0]: pin.split("==")[1] for pin in PLATFORM_PINS} self.assertGreaterEqual(Version(pins["protobuf"]), Version("6.31.1")) From 88ed9715e88b7053624d6e022ed8633744e47331 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 22 Sep 2026 20:23:02 -0700 Subject: [PATCH 2/7] lowering the floor on redis and not failing on new releases --- packages/core/canyonos_core/cli.py | 12 ++ .../controller/utils/redis_client.py | 5 +- packages/core/canyonos_core/stub_generator.py | 116 ++++++++---- packages/core/tests/test_cli.py | 23 +++ packages/core/tests/test_stub_generator.py | 176 +++++++++++++----- 5 files changed, 252 insertions(+), 80 deletions(-) diff --git a/packages/core/canyonos_core/cli.py b/packages/core/canyonos_core/cli.py index 1078d5de..fd2d3586 100644 --- a/packages/core/canyonos_core/cli.py +++ b/packages/core/canyonos_core/cli.py @@ -335,6 +335,18 @@ def _run_build(config_path): "Cannot build configured sources: " + "; ".join(missing_sources) ) + from canyonos_core.stub_generator import unsupported_requirements + + too_old = [ + f"{agent['name']} currently requires {asked}, but CanyonOS only supports {supported}" + for agent in agents + for asked, supported in unsupported_requirements(_normalize_requirements(agent)) + ] + for message in too_old: + logger.error("%s", message) + if too_old: + sys.exit(1) + # -------------------------------------------------------------- # # Step 1: Discover agent YAML files and generate Python stubs # # -------------------------------------------------------------- # diff --git a/packages/core/canyonos_core/controller/utils/redis_client.py b/packages/core/canyonos_core/controller/utils/redis_client.py index 3ba079e8..97bd62b0 100644 --- a/packages/core/canyonos_core/controller/utils/redis_client.py +++ b/packages/core/canyonos_core/controller/utils/redis_client.py @@ -30,7 +30,10 @@ def setnx(self, key, value): def expire(self, key, seconds, nx=False): """Set a TTL (in seconds) on a key. No-op if the key does not exist.""" - return self.client.expire(key, seconds, nx=nx) + # Raw command because redis-py only takes nx= from 4.2. + return self.client.execute_command( + "EXPIRE", key, seconds, *(["NX"] if nx else []) + ) # --- Hash operations --- diff --git a/packages/core/canyonos_core/stub_generator.py b/packages/core/canyonos_core/stub_generator.py index ca8e7c2c..e0bd079e 100644 --- a/packages/core/canyonos_core/stub_generator.py +++ b/packages/core/canyonos_core/stub_generator.py @@ -16,27 +16,35 @@ import shutil import yaml from packaging.requirements import InvalidRequirement, Requirement -from packaging.version import Version +from packaging.version import InvalidVersion, Version -# Packages every agent container needs; protobuf must be at least the gencode version of any *_pb2.py in the image. +# Lowest versions the image's own code runs on; an app asking for older fails the install. BASE_AGENT_REQUIREMENTS = [ - "grpcio==1.83.1", - "protobuf==6.33.5", - "redis==8.1.0", - "flask==3.1.3", - "requests==2.34.2", + "grpcio>=1.76.0", + "protobuf>=6.31.1", + "redis>=3.5", + "flask>=2.3.3", + "requests>=2.25", ] +# Newest major version of each base package CanyonOS is tested on; newer installs with a warning. +TESTED_MAJOR_VERSIONS = { + "grpcio": 1, + "protobuf": 6, + "redis": 8, + "flask": 3, + "requests": 2, +} + # Workflow containers currently need nothing beyond the base agent requirements # (telemetry and session state moved to Redis/OTLP, so no SQL driver is required). BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + [] -# Packages the image's own code is built against, so an app cannot be left to -# pick them alone. -_FORCED_FROM_BASE = ("protobuf", "grpcio", "requests") -PLATFORM_PINS = [ - pin for pin in BASE_AGENT_REQUIREMENTS if pin.split("==")[0] in _FORCED_FROM_BASE -] +# Every *_pb2.py checks this floor at import, which no package metadata carries, +# so it is forced past transitive bounds rather than left to the resolver. +PROTOBUF_FLOOR = Requirement( + next(pin for pin in BASE_AGENT_REQUIREMENTS if pin.startswith("protobuf")) +) def _build_import_nodes(): @@ -536,38 +544,61 @@ def _copy_files(output_dir, files_to_copy): def _platform_overrides(requirements): - """Take the higher of each platform pin and what the app asked for. + """Force the protobuf floor, intersected with any bound the app itself declares. - uv replaces a requirement rather than intersecting it, so the comparison - cannot be left to the resolver. + uv replaces a requirement rather than intersecting it, so the app's own + protobuf bound is folded into the override instead of being dropped. """ - declared = {} + specifier = PROTOBUF_FLOOR.specifier for requirement in requirements: try: parsed = Requirement(requirement) except InvalidRequirement: continue - declared[parsed.name.lower()] = parsed - - overrides = [] - for pin in PLATFORM_PINS: - name, pinned = pin.split("==") - asked = declared.get(name) - if asked is None or asked.specifier.contains(Version(pinned)): - overrides.append(pin) + if parsed.name.lower() == PROTOBUF_FLOOR.name: + specifier &= parsed.specifier + return [f"{PROTOBUF_FLOOR.name}{specifier}"] + + +def _caps_below(spec, floor): + """Whether this one specifier allows no version at or above floor.""" + try: + if spec.operator == "==" and spec.version.endswith(".*"): + release = Version(spec.version[:-2]).release + return ( + Version(".".join(map(str, (*release[:-1], release[-1] + 1)))) <= floor + ) + version = Version(spec.version) + except InvalidVersion: + return False + if spec.operator in ("==", "==="): + return version < floor + if spec.operator == "<": + return version <= floor + if spec.operator == "<=": + return version < floor + if spec.operator == "~=": + release = version.release + return Version(".".join(map(str, (*release[:-2], release[-2] + 1)))) <= floor + return False + + +def unsupported_requirements(requirements): + """(asked, supported) for each requirement that rules out every base-package version CanyonOS supports.""" + floors = {} + for base in BASE_AGENT_REQUIREMENTS: + parsed = Requirement(base) + floors[parsed.name] = (Version(next(iter(parsed.specifier)).version), base) + unsupported = [] + for requirement in requirements: + try: + parsed = Requirement(requirement) + except InvalidRequirement: continue - wanted = f"{asked.name}{asked.specifier}" - if any( - spec.operator in (">=", ">", "==", "~=") - and Version(spec.version.rstrip(".*")) > Version(pinned) - for spec in asked.specifier - ): - overrides.append(wanted) - print(f" Note: '{wanted}' outranks the platform pin {pin}") - else: - overrides.append(pin) - print(f" Warning: the platform pin {pin} breaks '{wanted}'") - return overrides + floor = floors.get(parsed.name.lower()) + if floor and any(_caps_below(spec, floor[0]) for spec in parsed.specifier): + unsupported.append((requirement, floor[1])) + return unsupported def _dependency_stage(overrides): @@ -579,9 +610,20 @@ def _dependency_stage(overrides): RUN --mount=type=cache,target=/root/.cache/uv printf '%s\\n' {forced} > /tmp/overrides.txt \\ && uv pip install --system -r requirements.txt --overrides /tmp/overrides.txt RUN uv pip check --system || echo "NOTE: CanyonOS forces {forced}; an incompatibility above naming one of those is a bound it could not share with the app." +RUN python -c "{_untested_version_check()}" """ +def _untested_version_check(): + """One-line Python that warns for each base package installed past its tested major version.""" + return ( + "import importlib.metadata as m; " + f"tested = {TESTED_MAJOR_VERSIONS!r}; " + "[print(f'WARNING: {n} {m.version(n)} is newer than CanyonOS has tested (up to {t}.x) and may not work.') " + "for n, t in tested.items() if int(m.version(n).split('.')[0]) > t]" + ) + + def generate_docker( yaml_path, agent_file, diff --git a/packages/core/tests/test_cli.py b/packages/core/tests/test_cli.py index 3bda9eea..3d44437a 100644 --- a/packages/core/tests/test_cli.py +++ b/packages/core/tests/test_cli.py @@ -236,6 +236,29 @@ def _write_agent_and_workflow_config(self, project_dir): ) return agent_yaml + def test_build_stops_on_a_requirement_below_the_supported_floor(self): + with tempfile.TemporaryDirectory() as tmpdir: + project_dir = Path(tmpdir) + agent_yaml = self._write_agent_and_workflow_config(project_dir) + config_path = project_dir / "config" / "global_controller.yaml" + config = yaml.safe_load(config_path.read_text()) + config["agents"][0]["requirements"] = ["flask==1.9"] + config_path.write_text(yaml.safe_dump(config)) + + with ( + self.assertRaises(SystemExit), + self.assertLogs("canyonos_core", level="ERROR") as logs, + ): + self._run_build(project_dir, [str(agent_yaml)], buildx_available=True) + + self.assertEqual( + logs.output, + [ + "ERROR:canyonos_core:ExampleAgent currently requires flask==1.9, " + "but CanyonOS only supports flask>=2.3.3" + ], + ) + def test_build_falls_back_to_sequential_docker_build_without_buildx(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) diff --git a/packages/core/tests/test_stub_generator.py b/packages/core/tests/test_stub_generator.py index 81fc2e03..f4ad75d6 100644 --- a/packages/core/tests/test_stub_generator.py +++ b/packages/core/tests/test_stub_generator.py @@ -3,11 +3,12 @@ import sys import tempfile import unittest +from unittest import mock from contextlib import redirect_stdout from pathlib import Path import yaml -from packaging.version import Version +from packaging.requirements import Requirement sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) @@ -15,7 +16,8 @@ from canyonos_core.stub_generator import ( BASE_AGENT_REQUIREMENTS, BASE_WORKFLOW_REQUIREMENTS, - PLATFORM_PINS, + PROTOBUF_FLOOR, + TESTED_MAJOR_VERSIONS, _stub_destination, _sweep_project_files, generate_docker, @@ -52,11 +54,11 @@ def test_base_only_when_requirements_omitted(self): self.assertEqual( requirements, [ - "grpcio==1.83.1", - "protobuf==6.33.5", - "redis==8.1.0", - "flask==3.1.3", - "requests==2.34.2", + "grpcio>=1.76.0", + "protobuf>=6.31.1", + "redis>=3.5", + "flask>=2.3.3", + "requests>=2.25", ], ) self.assertNotIn("yfinance", requirements) @@ -479,7 +481,7 @@ def test_stub_lands_both_flat_and_at_its_entrypoint_path(self): class PlatformPinTests(unittest.TestCase): - """Each forced package resolves to the higher of our pin and the app's ask.""" + """Only protobuf is forced, intersected with whatever bound the app declares.""" def _context(self, requirements, workflow=False): with tempfile.TemporaryDirectory() as tmpdir: @@ -517,54 +519,145 @@ def _context(self, requirements, workflow=False): ] return [entry.strip("'") for entry in written.split()], notes - def test_every_platform_pin_is_exact(self): - for pin in PLATFORM_PINS: - with self.subTest(pin=pin): - self.assertRegex(pin, r"^[a-z0-9-]+==[0-9][0-9a-z.]*$") + def test_base_requirements_are_ranges_not_exact_pins(self): + for requirement in BASE_AGENT_REQUIREMENTS: + with self.subTest(requirement=requirement): + self.assertNotIn("==", requirement) - def test_pins_come_from_the_base_requirements(self): - forced = {pin.split("==")[0] for pin in PLATFORM_PINS} - self.assertEqual(forced, set(stub_generator._FORCED_FROM_BASE)) - for pin in PLATFORM_PINS: - with self.subTest(pin=pin): - self.assertIn(pin, BASE_AGENT_REQUIREMENTS) - - def test_the_forced_protobuf_loads_host_compiled_gencode(self): + def test_the_protobuf_floor_loads_host_compiled_gencode(self): # The host's grpcio-tools 1.76.0 emits *_pb2.py that need protobuf>=6.31.1 at runtime. - pins = {pin.split("==")[0]: pin.split("==")[1] for pin in PLATFORM_PINS} - self.assertGreaterEqual(Version(pins["protobuf"]), Version("6.31.1")) + self.assertIn("6.31.1", PROTOBUF_FLOOR.specifier) + self.assertNotIn("6.31.0", PROTOBUF_FLOOR.specifier) - def test_the_pin_holds_and_stays_quiet_when_nothing_newer_is_asked(self): + def test_only_protobuf_is_forced(self): for requirements in ( [], - ["protobuf>=5.29.0"], - ["protobuf==6.33.5"], ["streamlit==1.31.1"], + ["requests==2.28.0", "flask==2.3.3", "grpcio==1.80.0"], ): with self.subTest(requirements=requirements): overrides, notes = self._context(requirements) - self.assertEqual(overrides, list(PLATFORM_PINS)) + self.assertEqual(overrides, [str(PROTOBUF_FLOOR)]) self.assertEqual(notes, []) - def test_an_app_asking_for_newer_wins(self): - overrides, notes = self._context(["protobuf>=7"]) - self.assertIn("protobuf>=7", overrides) - self.assertNotIn("protobuf==6.33.5", overrides) + def test_an_app_protobuf_bound_is_intersected_with_the_floor(self): + overrides, _ = self._context(["protobuf>=6.32"]) + self.assertEqual(len(overrides), 1) + forced = Requirement(overrides[0]).specifier + self.assertNotIn("6.31.5", forced) + self.assertIn("6.33.5", forced) + + def test_other_base_packages_are_left_to_the_resolver(self): + with tempfile.TemporaryDirectory() as tmpdir: + yaml_path, agent_file = GenerateDockerRequirementsTests._write_agent_yaml( + self, tmpdir + ) + output_dir = os.path.join(tmpdir, "out") + with redirect_stdout(io.StringIO()): + generate_docker( + yaml_path, + agent_file, + output_dir=output_dir, + requirements=["flask==2.3.3"], + ) + requirements = _read_requirements(output_dir) + + self.assertIn("flask>=2.3.3", requirements) + self.assertIn("flask==2.3.3", requirements) + + def test_base_requirements_carry_no_upper_bound(self): + for requirement in BASE_AGENT_REQUIREMENTS: + with self.subTest(requirement=requirement): + self.assertNotIn("<", requirement) + + def test_every_base_package_has_a_tested_major_version(self): + names = {Requirement(r).name for r in BASE_AGENT_REQUIREMENTS} + self.assertEqual(names, set(TESTED_MAJOR_VERSIONS)) + + def test_the_untested_version_check_warns_only_past_the_tested_major(self): + check = stub_generator._untested_version_check() + installed = { + "grpcio": "1.80.0", + "protobuf": "7.0.1", + "redis": "8.1.0", + "flask": "3.1.3", + "requests": "2.34.2", + } + buffer = io.StringIO() + with mock.patch( + "importlib.metadata.version", side_effect=installed.__getitem__ + ): + with redirect_stdout(buffer): + exec(check, {}) self.assertEqual( - notes, ["Note: 'protobuf>=7' outranks the platform pin protobuf==6.33.5"] + buffer.getvalue().splitlines(), + [ + "WARNING: protobuf 7.0.1 is newer than CanyonOS has tested (up to 6.x) and may not work." + ], ) - def test_an_app_asking_for_older_loses_and_is_told(self): - overrides, notes = self._context(["protobuf<5"]) - self.assertIn("protobuf==6.33.5", overrides) + def test_the_untested_version_check_runs_in_both_dockerfiles(self): + for workflow in (False, True): + with self.subTest(workflow=workflow): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + output_dir = os.path.join(tmpdir, "out") + with redirect_stdout(io.StringIO()): + if workflow: + wf = _write(project / "workflow.py", "print('ok')\n") + generate_workflow_docker(str(wf), [], output_dir=output_dir) + else: + yaml_path = project / "ExampleAgent.yaml" + yaml_path.write_text( + yaml.safe_dump({"agent": {"name": "ExampleAgent"}}) + ) + agent = _write(project / "agent.py", "print('ok')\n") + generate_docker( + str(yaml_path), str(agent), output_dir=output_dir + ) + dockerfile = _read_dockerfile(output_dir) + self.assertIn( + f'RUN python -c "{stub_generator._untested_version_check()}"', + dockerfile, + ) + + def test_requirements_below_a_floor_are_reported(self): + self.assertEqual( + stub_generator.unsupported_requirements( + [ + "flask==1.9", + "flask<2.3", + "flask~=2.2.0", + "flask==2.2.*", + "protobuf<5", + ] + ), + [ + ("flask==1.9", "flask>=2.3.3"), + ("flask<2.3", "flask>=2.3.3"), + ("flask~=2.2.0", "flask>=2.3.3"), + ("flask==2.2.*", "flask>=2.3.3"), + ("protobuf<5", "protobuf>=6.31.1"), + ], + ) + + def test_requirements_that_reach_a_floor_are_not_reported(self): self.assertEqual( - notes, ["Warning: the platform pin protobuf==6.33.5 breaks 'protobuf<5'"] + stub_generator.unsupported_requirements( + [ + "flask~=2.2", + "flask>=2", + "flask!=2.3.3", + "flask==2.3.3", + "yfinance==0.1", + ] + ), + [], ) def test_the_workflow_context_decides_the_same_way(self): - overrides, notes = self._context(["protobuf>=7"], workflow=True) - self.assertIn("protobuf>=7", overrides) - self.assertEqual(len(notes), 1) + overrides, _ = self._context(["protobuf>=6.32"], workflow=True) + self.assertEqual(overrides, self._context(["protobuf>=6.32"])[0]) def test_override_entries_are_quoted_for_the_shell(self): # An unquoted `protobuf>=7` would be a redirect, not an argument. @@ -579,11 +672,11 @@ def test_override_entries_are_quoted_for_the_shell(self): str(yaml_path), str(agent_file), output_dir=output_dir, - requirements=["protobuf>=7"], + requirements=["protobuf>=6.32"], ) dockerfile = _read_dockerfile(output_dir) - self.assertIn("'protobuf>=7'", dockerfile) + self.assertIn("'protobuf>=6.31.1,>=6.32'", dockerfile) def test_both_dockerfiles_install_with_the_overrides_and_report(self): for workflow in (False, True): @@ -609,8 +702,7 @@ def test_both_dockerfiles_install_with_the_overrides_and_report(self): install = dockerfile.split("RUN uv pip check")[0] self.assertIn("--overrides /tmp/overrides.txt", install) self.assertIn("uv pip check --system", dockerfile) - for pin in PLATFORM_PINS: - self.assertIn(pin, dockerfile.split("NOTE:")[1]) + self.assertIn(str(PROTOBUF_FLOOR), dockerfile.split("NOTE:")[1]) if __name__ == "__main__": From 0c14d7f2007bf0cbf2e3b576753f155d628a94bc Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 22 Sep 2026 20:31:47 -0700 Subject: [PATCH 3/7] readme --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f2d7b2f..e6e91ac5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- CanyonOS + CanyonOS

## CanyonOS turns plain Python into a running, distributed workflow — without changing a line of code. @@ -138,6 +138,15 @@ canyonos test "Hello World!" --json ### 3. Deploy +
+Supported dependency versions + +If your code imports any of these, it must allow at least this version. Older isn't supported, sorry! + +`grpcio>=1.76.0` · `protobuf>=6.31.1` · `redis>=3.5` · `flask>=2.3.3` · `requests>=2.25` + +
+ Deploy the project fully, configured by the config files. On deploy success, a `POST` endpoint will be returned, in which you can send your workflow queries to. From 68df188cadb0c1083bdbbf537d97b772bf2db8fe Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 22 Sep 2026 20:46:55 -0700 Subject: [PATCH 4/7] moving requirements into proxy to simplify requirements --- .../porting-to-canyonos/validation/runtime.py | 4 +- packages/core/canyonos_core/cli.py | 13 ++- .../controller/local_controller.py | 16 ++- packages/core/canyonos_core/stub_generator.py | 36 +++--- packages/core/tests/test_cli.py | 5 +- packages/core/tests/test_stub_generator.py | 104 ++++++++++++------ 6 files changed, 119 insertions(+), 59 deletions(-) diff --git a/.claude/skills/porting-to-canyonos/validation/runtime.py b/.claude/skills/porting-to-canyonos/validation/runtime.py index 2c6199c5..e92d1000 100644 --- a/.claude/skills/porting-to-canyonos/validation/runtime.py +++ b/.claude/skills/porting-to-canyonos/validation/runtime.py @@ -48,8 +48,8 @@ def _base_requirements(): - agent = ["grpcio", "protobuf", "redis", "flask", "requests"] - workflow = [*agent] + agent = ["grpcio", "protobuf", "redis"] + workflow = [*agent, "flask"] try: from canyonos_core import stub_generator except Exception: # noqa: BLE001 - a broken install must not crash validation diff --git a/packages/core/canyonos_core/cli.py b/packages/core/canyonos_core/cli.py index fd2d3586..8138882c 100644 --- a/packages/core/canyonos_core/cli.py +++ b/packages/core/canyonos_core/cli.py @@ -335,12 +335,21 @@ def _run_build(config_path): "Cannot build configured sources: " + "; ".join(missing_sources) ) - from canyonos_core.stub_generator import unsupported_requirements + from canyonos_core.stub_generator import ( + BASE_AGENT_REQUIREMENTS, + BASE_WORKFLOW_REQUIREMENTS, + unsupported_requirements, + ) too_old = [ f"{agent['name']} currently requires {asked}, but CanyonOS only supports {supported}" for agent in agents - for asked, supported in unsupported_requirements(_normalize_requirements(agent)) + for asked, supported in unsupported_requirements( + _normalize_requirements(agent), + BASE_WORKFLOW_REQUIREMENTS + if agent.get("type", "agent") == "workflow" + else BASE_AGENT_REQUIREMENTS, + ) ] for message in too_old: logger.error("%s", message) diff --git a/packages/core/canyonos_core/controller/local_controller.py b/packages/core/canyonos_core/controller/local_controller.py index 703a18f3..1a8a40b1 100644 --- a/packages/core/canyonos_core/controller/local_controller.py +++ b/packages/core/canyonos_core/controller/local_controller.py @@ -57,6 +57,7 @@ import local_controler_pb2 import local_controler_pb2_grpc + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -198,8 +199,7 @@ def _start_llm_proxy(self, redis_host, redis_port): """ import socket import subprocess - - import requests + import urllib.request # An orphaned proxy on 8081 would answer the /healthz probe below and mask # one of ours that never bound, so prove the port free before spawning. @@ -224,9 +224,13 @@ def _start_llm_proxy(self, redis_host, redis_port): "CANYONOS_REDIS_PORT": str(redis_port), } ) + # The image gives the proxy its own venv so its packages never share versions with the agent's. + proxy_python = "/opt/canyonos-proxy/bin/python" + if not os.path.exists(proxy_python): + proxy_python = sys.executable try: proxy_process = subprocess.Popen( - [sys.executable, "-m", "canyonos_core.llm_proxy"], + [proxy_python, "-m", "canyonos_core.llm_proxy"], env=proxy_env, ) except Exception as e: @@ -247,9 +251,11 @@ def _start_llm_proxy(self, redis_host, redis_port): "otherwise fail silently." ) try: - if requests.get("http://127.0.0.1:8081/healthz", timeout=0.5).ok: + with urllib.request.urlopen( + "http://127.0.0.1:8081/healthz", timeout=0.5 + ): break - except requests.exceptions.RequestException: + except OSError: pass time.sleep(0.2) else: diff --git a/packages/core/canyonos_core/stub_generator.py b/packages/core/canyonos_core/stub_generator.py index e0bd079e..c5073a71 100644 --- a/packages/core/canyonos_core/stub_generator.py +++ b/packages/core/canyonos_core/stub_generator.py @@ -23,8 +23,6 @@ "grpcio>=1.76.0", "protobuf>=6.31.1", "redis>=3.5", - "flask>=2.3.3", - "requests>=2.25", ] # Newest major version of each base package CanyonOS is tested on; newer installs with a warning. @@ -33,12 +31,17 @@ "protobuf": 6, "redis": 8, "flask": 3, - "requests": 2, } -# Workflow containers currently need nothing beyond the base agent requirements -# (telemetry and session state moved to Redis/OTLP, so no SQL driver is required). -BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + [] +# deploy.py serves the workflow's HTTP API from the workflow's own process. +BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask>=2.3.3"] + +# The LLM proxy runs from its own venv, so these exact pins never meet the app's. +# These requirements are not pinned to a specific version because LLM-proxy is completely managed by CanyonOS, with no user code interacting with the internals +PROXY_REQUIREMENTS = ["flask==3.1.3", "requests==2.34.2", "redis==8.1.0"] + +# Only images whose app can import boto3 can call Bedrock, so only they get the proxy's Bedrock route. +PROXY_BEDROCK_REQUIREMENT = "boto3==1.43.91" # Every *_pb2.py checks this floor at import, which no package metadata carries, # so it is forced past transitive bounds rather than left to the resolver. @@ -583,10 +586,10 @@ def _caps_below(spec, floor): return False -def unsupported_requirements(requirements): +def unsupported_requirements(requirements, base_requirements=BASE_AGENT_REQUIREMENTS): """(asked, supported) for each requirement that rules out every base-package version CanyonOS supports.""" floors = {} - for base in BASE_AGENT_REQUIREMENTS: + for base in base_requirements: parsed = Requirement(base) floors[parsed.name] = (Version(next(iter(parsed.specifier)).version), base) unsupported = [] @@ -601,7 +604,7 @@ def unsupported_requirements(requirements): return unsupported -def _dependency_stage(overrides): +def _dependency_stage(overrides, base_requirements): """Render the install stage. uv reads overrides from a file and takes no inline form, so the image writes one; the entries are quoted because a bare `>=` would be a redirect.""" @@ -610,15 +613,20 @@ def _dependency_stage(overrides): RUN --mount=type=cache,target=/root/.cache/uv printf '%s\\n' {forced} > /tmp/overrides.txt \\ && uv pip install --system -r requirements.txt --overrides /tmp/overrides.txt RUN uv pip check --system || echo "NOTE: CanyonOS forces {forced}; an incompatibility above naming one of those is a bound it could not share with the app." -RUN python -c "{_untested_version_check()}" +RUN python -c "{_untested_version_check(base_requirements)}" +RUN --mount=type=cache,target=/root/.cache/uv uv venv /opt/canyonos-proxy \\ + && uv pip install --python /opt/canyonos-proxy/bin/python {" ".join(PROXY_REQUIREMENTS)} \\ + && if python -c "import boto3" 2>/dev/null; then uv pip install --python /opt/canyonos-proxy/bin/python {PROXY_BEDROCK_REQUIREMENT}; fi """ -def _untested_version_check(): +def _untested_version_check(base_requirements): """One-line Python that warns for each base package installed past its tested major version.""" + names = {Requirement(r).name for r in base_requirements} + tested = {n: t for n, t in TESTED_MAJOR_VERSIONS.items() if n in names} return ( "import importlib.metadata as m; " - f"tested = {TESTED_MAJOR_VERSIONS!r}; " + f"tested = {tested!r}; " "[print(f'WARNING: {n} {m.version(n)} is newer than CanyonOS has tested (up to {t}.x) and may not work.') " "for n, t in tested.items() if int(m.version(n).split('.')[0]) > t]" ) @@ -760,7 +768,7 @@ def generate_docker( ENV PYTHONUNBUFFERED=1 -{_dependency_stage(overrides)} +{_dependency_stage(overrides, BASE_AGENT_REQUIREMENTS)} COPY . . ENV CANYONOS_AGENT_NAME={agent_name} @@ -943,7 +951,7 @@ def mark_ready_when_serving(): ENV PYTHONUNBUFFERED=1 -{_dependency_stage(overrides)} +{_dependency_stage(overrides, BASE_WORKFLOW_REQUIREMENTS)} COPY . . EXPOSE 50051 diff --git a/packages/core/tests/test_cli.py b/packages/core/tests/test_cli.py index 3d44437a..5716b623 100644 --- a/packages/core/tests/test_cli.py +++ b/packages/core/tests/test_cli.py @@ -236,13 +236,14 @@ def _write_agent_and_workflow_config(self, project_dir): ) return agent_yaml - def test_build_stops_on_a_requirement_below_the_supported_floor(self): + def test_build_stops_on_a_workflow_requirement_below_the_supported_floor(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) agent_yaml = self._write_agent_and_workflow_config(project_dir) config_path = project_dir / "config" / "global_controller.yaml" config = yaml.safe_load(config_path.read_text()) config["agents"][0]["requirements"] = ["flask==1.9"] + config["agents"][1]["requirements"] = ["flask==1.9"] config_path.write_text(yaml.safe_dump(config)) with ( @@ -254,7 +255,7 @@ def test_build_stops_on_a_requirement_below_the_supported_floor(self): self.assertEqual( logs.output, [ - "ERROR:canyonos_core:ExampleAgent currently requires flask==1.9, " + "ERROR:canyonos_core:Workflow currently requires flask==1.9, " "but CanyonOS only supports flask>=2.3.3" ], ) diff --git a/packages/core/tests/test_stub_generator.py b/packages/core/tests/test_stub_generator.py index f4ad75d6..8489d366 100644 --- a/packages/core/tests/test_stub_generator.py +++ b/packages/core/tests/test_stub_generator.py @@ -57,8 +57,6 @@ def test_base_only_when_requirements_omitted(self): "grpcio>=1.76.0", "protobuf>=6.31.1", "redis>=3.5", - "flask>=2.3.3", - "requests>=2.25", ], ) self.assertNotIn("yfinance", requirements) @@ -549,14 +547,12 @@ def test_an_app_protobuf_bound_is_intersected_with_the_floor(self): def test_other_base_packages_are_left_to_the_resolver(self): with tempfile.TemporaryDirectory() as tmpdir: - yaml_path, agent_file = GenerateDockerRequirementsTests._write_agent_yaml( - self, tmpdir - ) + workflow_file = _write(Path(tmpdir) / "workflow.py", "print('ok')\n") output_dir = os.path.join(tmpdir, "out") with redirect_stdout(io.StringIO()): - generate_docker( - yaml_path, - agent_file, + generate_workflow_docker( + str(workflow_file), + [], output_dir=output_dir, requirements=["flask==2.3.3"], ) @@ -566,22 +562,21 @@ def test_other_base_packages_are_left_to_the_resolver(self): self.assertIn("flask==2.3.3", requirements) def test_base_requirements_carry_no_upper_bound(self): - for requirement in BASE_AGENT_REQUIREMENTS: + for requirement in BASE_WORKFLOW_REQUIREMENTS: with self.subTest(requirement=requirement): self.assertNotIn("<", requirement) def test_every_base_package_has_a_tested_major_version(self): - names = {Requirement(r).name for r in BASE_AGENT_REQUIREMENTS} + names = {Requirement(r).name for r in BASE_WORKFLOW_REQUIREMENTS} self.assertEqual(names, set(TESTED_MAJOR_VERSIONS)) def test_the_untested_version_check_warns_only_past_the_tested_major(self): - check = stub_generator._untested_version_check() + check = stub_generator._untested_version_check(BASE_WORKFLOW_REQUIREMENTS) installed = { "grpcio": "1.80.0", "protobuf": "7.0.1", "redis": "8.1.0", "flask": "3.1.3", - "requests": "2.34.2", } buffer = io.StringIO() with mock.patch( @@ -596,29 +591,65 @@ def test_the_untested_version_check_warns_only_past_the_tested_major(self): ], ) - def test_the_untested_version_check_runs_in_both_dockerfiles(self): + def _dockerfile(self, workflow, requirements=None): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + output_dir = os.path.join(tmpdir, "out") + with redirect_stdout(io.StringIO()): + if workflow: + wf = _write(project / "workflow.py", "print('ok')\n") + generate_workflow_docker( + str(wf), [], output_dir=output_dir, requirements=requirements + ) + else: + yaml_path = project / "ExampleAgent.yaml" + yaml_path.write_text( + yaml.safe_dump({"agent": {"name": "ExampleAgent"}}) + ) + agent = _write(project / "agent.py", "print('ok')\n") + generate_docker( + str(yaml_path), + str(agent), + output_dir=output_dir, + requirements=requirements, + ) + return _read_dockerfile(output_dir) + + def test_the_untested_version_check_covers_only_that_images_base(self): + for workflow, base in ( + (False, BASE_AGENT_REQUIREMENTS), + (True, BASE_WORKFLOW_REQUIREMENTS), + ): + with self.subTest(workflow=workflow): + self.assertIn( + f'RUN python -c "{stub_generator._untested_version_check(base)}"', + self._dockerfile(workflow), + ) + self.assertNotIn( + "'flask'", stub_generator._untested_version_check(BASE_AGENT_REQUIREMENTS) + ) + + def test_the_proxy_gets_its_own_venv_after_the_app_install(self): for workflow in (False, True): with self.subTest(workflow=workflow): - with tempfile.TemporaryDirectory() as tmpdir: - project = Path(tmpdir) - output_dir = os.path.join(tmpdir, "out") - with redirect_stdout(io.StringIO()): - if workflow: - wf = _write(project / "workflow.py", "print('ok')\n") - generate_workflow_docker(str(wf), [], output_dir=output_dir) - else: - yaml_path = project / "ExampleAgent.yaml" - yaml_path.write_text( - yaml.safe_dump({"agent": {"name": "ExampleAgent"}}) - ) - agent = _write(project / "agent.py", "print('ok')\n") - generate_docker( - str(yaml_path), str(agent), output_dir=output_dir - ) - dockerfile = _read_dockerfile(output_dir) + dockerfile = self._dockerfile(workflow) + app_install = dockerfile.index("uv pip install --system") + proxy_install = dockerfile.index("uv venv /opt/canyonos-proxy") + self.assertLess(app_install, proxy_install) + proxy_stage = dockerfile[proxy_install:] + for pin in stub_generator.PROXY_REQUIREMENTS: + self.assertIn(pin, proxy_stage) self.assertIn( - f'RUN python -c "{stub_generator._untested_version_check()}"', - dockerfile, + 'if python -c "import boto3" 2>/dev/null; then uv pip install ' + f"--python /opt/canyonos-proxy/bin/python {stub_generator.PROXY_BEDROCK_REQUIREMENT}; fi", + proxy_stage, + ) + + def test_agents_no_longer_carry_the_proxys_packages(self): + for name in ("flask", "requests", "boto3"): + with self.subTest(name=name): + self.assertFalse( + any(Requirement(r).name == name for r in BASE_AGENT_REQUIREMENTS) ) def test_requirements_below_a_floor_are_reported(self): @@ -630,7 +661,8 @@ def test_requirements_below_a_floor_are_reported(self): "flask~=2.2.0", "flask==2.2.*", "protobuf<5", - ] + ], + BASE_WORKFLOW_REQUIREMENTS, ), [ ("flask==1.9", "flask>=2.3.3"), @@ -650,11 +682,15 @@ def test_requirements_that_reach_a_floor_are_not_reported(self): "flask!=2.3.3", "flask==2.3.3", "yfinance==0.1", - ] + ], + BASE_WORKFLOW_REQUIREMENTS, ), [], ) + def test_an_agent_may_pin_any_flask_now_the_proxy_has_its_own(self): + self.assertEqual(stub_generator.unsupported_requirements(["flask==1.0"]), []) + def test_the_workflow_context_decides_the_same_way(self): overrides, _ = self._context(["protobuf>=6.32"], workflow=True) self.assertEqual(overrides, self._context(["protobuf>=6.32"])[0]) From e0ab554fab37a86e66c9d87c0d8674c96c97f916 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 22 Sep 2026 20:48:15 -0700 Subject: [PATCH 5/7] updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e6e91ac5..57ec8a8b 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ canyonos test "Hello World!" --json If your code imports any of these, it must allow at least this version. Older isn't supported, sorry! -`grpcio>=1.76.0` · `protobuf>=6.31.1` · `redis>=3.5` · `flask>=2.3.3` · `requests>=2.25` +`grpcio>=1.76.0` · `protobuf>=6.31.1` · `redis>=3.5` From 0a885c325f033b45ec10ccbf42e30cf4ef5500c0 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 22 Sep 2026 21:04:28 -0700 Subject: [PATCH 6/7] simplified --- packages/core/canyonos_core/cli.py | 26 +++-- .../controller/local_controller.py | 8 +- packages/core/canyonos_core/stub_generator.py | 80 ++++++++++----- packages/core/tests/test_cli.py | 21 ++++ .../test_local_controller_proxy_start.py | 41 ++++++++ packages/core/tests/test_stub_generator.py | 97 +++++++++++-------- 6 files changed, 200 insertions(+), 73 deletions(-) create mode 100644 packages/core/tests/test_local_controller_proxy_start.py diff --git a/packages/core/canyonos_core/cli.py b/packages/core/canyonos_core/cli.py index 8138882c..ec3dca20 100644 --- a/packages/core/canyonos_core/cli.py +++ b/packages/core/canyonos_core/cli.py @@ -338,19 +338,29 @@ def _run_build(config_path): from canyonos_core.stub_generator import ( BASE_AGENT_REQUIREMENTS, BASE_WORKFLOW_REQUIREMENTS, - unsupported_requirements, + too_new_requirements, + too_old_requirements, ) - too_old = [ - f"{agent['name']} currently requires {asked}, but CanyonOS only supports {supported}" - for agent in agents - for asked, supported in unsupported_requirements( - _normalize_requirements(agent), + too_old = [] + for agent in agents: + requirements = _normalize_requirements(agent) + base = ( BASE_WORKFLOW_REQUIREMENTS if agent.get("type", "agent") == "workflow" - else BASE_AGENT_REQUIREMENTS, + else BASE_AGENT_REQUIREMENTS ) - ] + too_old += [ + f"{agent['name']} currently requires {asked}, but CanyonOS only supports {supported}" + for asked, supported in too_old_requirements(requirements, base) + ] + for asked, tested in too_new_requirements(requirements, base): + logger.warning( + "%s currently requires %s, but CanyonOS has only tested %s; it may not work", + agent["name"], + asked, + tested, + ) for message in too_old: logger.error("%s", message) if too_old: diff --git a/packages/core/canyonos_core/controller/local_controller.py b/packages/core/canyonos_core/controller/local_controller.py index 1a8a40b1..bc4f44e8 100644 --- a/packages/core/canyonos_core/controller/local_controller.py +++ b/packages/core/canyonos_core/controller/local_controller.py @@ -243,6 +243,7 @@ def _start_llm_proxy(self, redis_host, redis_port): # Popen only raises if the process can't be spawned -- it returns a healthy # handle even if the proxy starts and dies immediately, so poll /healthz. deadline = time.time() + 10 + last_error = None while time.time() < deadline: if proxy_process.poll() is not None: raise RuntimeError( @@ -255,13 +256,14 @@ def _start_llm_proxy(self, redis_host, redis_port): "http://127.0.0.1:8081/healthz", timeout=0.5 ): break - except OSError: - pass + except OSError as e: + last_error = e time.sleep(0.2) else: proxy_process.kill() raise RuntimeError( - "LLM proxy did not become healthy on 127.0.0.1:8081 within 10s; " + "LLM proxy did not become healthy on 127.0.0.1:8081 within 10s " + f"(last health check: {last_error}); " "agent LLM calls are routed through it unconditionally and would " "otherwise fail silently." ) diff --git a/packages/core/canyonos_core/stub_generator.py b/packages/core/canyonos_core/stub_generator.py index c5073a71..c77ce330 100644 --- a/packages/core/canyonos_core/stub_generator.py +++ b/packages/core/canyonos_core/stub_generator.py @@ -25,7 +25,7 @@ "redis>=3.5", ] -# Newest major version of each base package CanyonOS is tested on; newer installs with a warning. +# Newest major version of each base package CanyonOS is tested on; installs stay below the next major unless the app asks for newer, which only warns. TESTED_MAJOR_VERSIONS = { "grpcio": 1, "protobuf": 6, @@ -586,8 +586,8 @@ def _caps_below(spec, floor): return False -def unsupported_requirements(requirements, base_requirements=BASE_AGENT_REQUIREMENTS): - """(asked, supported) for each requirement that rules out every base-package version CanyonOS supports.""" +def too_old_requirements(requirements, base_requirements=BASE_AGENT_REQUIREMENTS): + """Return (requirement, our_floor) for each requirement that only allows versions older than CanyonOS supports.""" floors = {} for base in base_requirements: parsed = Requirement(base) @@ -604,34 +604,68 @@ def unsupported_requirements(requirements, base_requirements=BASE_AGENT_REQUIREM return unsupported -def _dependency_stage(overrides, base_requirements): - """Render the install stage. uv reads overrides from a file and takes no - inline form, so the image writes one; the entries are quoted because a bare - `>=` would be a redirect.""" +def _forces_at_or_above(spec, limit): + """Whether this one specifier allows only versions at or above limit.""" + try: + if spec.operator == "==" and spec.version.endswith(".*"): + return Version(spec.version[:-2]) >= limit + version = Version(spec.version) + except InvalidVersion: + return False + return spec.operator in ("==", "===", ">=", ">", "~=") and version >= limit + + +def too_new_requirements(requirements, base_requirements=BASE_AGENT_REQUIREMENTS): + """Return (requirement, tested_limit) for each requirement that only allows versions newer than CanyonOS has tested.""" + limits = { + name: Version(str(major + 1)) + for name, major in _tested_majors(base_requirements).items() + } + too_new = [] + for requirement in requirements: + try: + parsed = Requirement(requirement) + except InvalidRequirement: + continue + limit = limits.get(parsed.name.lower()) + if limit and any(_forces_at_or_above(spec, limit) for spec in parsed.specifier): + too_new.append((requirement, f"{parsed.name.lower()}<{limit}")) + return too_new + + +def _tested_majors(base_requirements): + """TESTED_MAJOR_VERSIONS narrowed to the packages this image's base list installs.""" + names = {Requirement(r).name for r in base_requirements} + return {n: t for n, t in TESTED_MAJOR_VERSIONS.items() if n in names} + + +def _dependency_stage(overrides, base_requirements, requirements): + """Render the install stage: app packages held below the tested majors, then the proxy's own venv. + + uv reads overrides and constraints only from files, so the image writes them; + entries are quoted because a bare `>=` or `<` would be a redirect. + """ forced = " ".join(f"'{override}'" for override in overrides) + asked_newer = { + Requirement(requirement).name.lower() + for requirement, _ in too_new_requirements(requirements, base_requirements) + } + caps = " ".join( + f"'{name}<{major + 1}'" + for name, major in _tested_majors(base_requirements).items() + if name not in asked_newer + ) return f"""COPY requirements.txt . RUN --mount=type=cache,target=/root/.cache/uv printf '%s\\n' {forced} > /tmp/overrides.txt \\ - && uv pip install --system -r requirements.txt --overrides /tmp/overrides.txt + && printf '%s\\n' {caps} > /tmp/tested.txt \\ + && uv pip install --system -r requirements.txt --overrides /tmp/overrides.txt -c /tmp/tested.txt RUN uv pip check --system || echo "NOTE: CanyonOS forces {forced}; an incompatibility above naming one of those is a bound it could not share with the app." -RUN python -c "{_untested_version_check(base_requirements)}" RUN --mount=type=cache,target=/root/.cache/uv uv venv /opt/canyonos-proxy \\ && uv pip install --python /opt/canyonos-proxy/bin/python {" ".join(PROXY_REQUIREMENTS)} \\ && if python -c "import boto3" 2>/dev/null; then uv pip install --python /opt/canyonos-proxy/bin/python {PROXY_BEDROCK_REQUIREMENT}; fi """ -def _untested_version_check(base_requirements): - """One-line Python that warns for each base package installed past its tested major version.""" - names = {Requirement(r).name for r in base_requirements} - tested = {n: t for n, t in TESTED_MAJOR_VERSIONS.items() if n in names} - return ( - "import importlib.metadata as m; " - f"tested = {tested!r}; " - "[print(f'WARNING: {n} {m.version(n)} is newer than CanyonOS has tested (up to {t}.x) and may not work.') " - "for n, t in tested.items() if int(m.version(n).split('.')[0]) > t]" - ) - - def generate_docker( yaml_path, agent_file, @@ -768,7 +802,7 @@ def generate_docker( ENV PYTHONUNBUFFERED=1 -{_dependency_stage(overrides, BASE_AGENT_REQUIREMENTS)} +{_dependency_stage(overrides, BASE_AGENT_REQUIREMENTS, requirements or [])} COPY . . ENV CANYONOS_AGENT_NAME={agent_name} @@ -951,7 +985,7 @@ def mark_ready_when_serving(): ENV PYTHONUNBUFFERED=1 -{_dependency_stage(overrides, BASE_WORKFLOW_REQUIREMENTS)} +{_dependency_stage(overrides, BASE_WORKFLOW_REQUIREMENTS, requirements or [])} COPY . . EXPOSE 50051 diff --git a/packages/core/tests/test_cli.py b/packages/core/tests/test_cli.py index 5716b623..d58dc996 100644 --- a/packages/core/tests/test_cli.py +++ b/packages/core/tests/test_cli.py @@ -260,6 +260,27 @@ def test_build_stops_on_a_workflow_requirement_below_the_supported_floor(self): ], ) + def test_build_warns_on_a_requirement_above_the_tested_major_and_continues(self): + with tempfile.TemporaryDirectory() as tmpdir: + project_dir = Path(tmpdir) + agent_yaml = self._write_agent_and_workflow_config(project_dir) + config_path = project_dir / "config" / "global_controller.yaml" + config = yaml.safe_load(config_path.read_text()) + config["agents"][0]["requirements"] = ["protobuf>=7"] + config_path.write_text(yaml.safe_dump(config)) + + with self.assertLogs("canyonos_core", level="WARNING") as logs: + _, generate_docker, _ = self._run_build( + project_dir, [str(agent_yaml)], buildx_available=True + ) + + self.assertIn( + "WARNING:canyonos_core:ExampleAgent currently requires protobuf>=7, " + "but CanyonOS has only tested protobuf<7; it may not work", + logs.output, + ) + generate_docker.assert_called_once() + def test_build_falls_back_to_sequential_docker_build_without_buildx(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) diff --git a/packages/core/tests/test_local_controller_proxy_start.py b/packages/core/tests/test_local_controller_proxy_start.py new file mode 100644 index 00000000..91a1ede5 --- /dev/null +++ b/packages/core/tests/test_local_controller_proxy_start.py @@ -0,0 +1,41 @@ +import unittest +import urllib.error +from unittest.mock import MagicMock, patch + +from canyonos_core.controller.local_controller import LocalController + + +class StartLlmProxyTests(unittest.TestCase): + def _start_with_health_check_failing(self, error): + proxy_process = MagicMock() + proxy_process.poll.return_value = None + fake_time = MagicMock() + fake_time.time.side_effect = [0, 0, 11] + with ( + patch("socket.socket"), + patch("subprocess.Popen", return_value=proxy_process), + patch("urllib.request.urlopen", side_effect=error), + patch("canyonos_core.controller.local_controller.time", fake_time), + self.assertRaises(RuntimeError) as raised, + ): + LocalController._start_llm_proxy(MagicMock(), "localhost", 6379) + proxy_process.kill.assert_called_once() + return str(raised.exception) + + def test_the_timeout_names_the_last_health_check_failure(self): + message = self._start_with_health_check_failing( + urllib.error.URLError(ConnectionRefusedError(111, "Connection refused")) + ) + self.assertIn("did not become healthy on 127.0.0.1:8081 within 10s", message) + self.assertIn( + "(last health check: )", + message, + ) + + def test_a_timed_out_health_check_is_named_too(self): + message = self._start_with_health_check_failing(TimeoutError("timed out")) + self.assertIn("(last health check: timed out)", message) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/core/tests/test_stub_generator.py b/packages/core/tests/test_stub_generator.py index 8489d366..cec8fbc4 100644 --- a/packages/core/tests/test_stub_generator.py +++ b/packages/core/tests/test_stub_generator.py @@ -3,7 +3,6 @@ import sys import tempfile import unittest -from unittest import mock from contextlib import redirect_stdout from pathlib import Path @@ -570,27 +569,6 @@ def test_every_base_package_has_a_tested_major_version(self): names = {Requirement(r).name for r in BASE_WORKFLOW_REQUIREMENTS} self.assertEqual(names, set(TESTED_MAJOR_VERSIONS)) - def test_the_untested_version_check_warns_only_past_the_tested_major(self): - check = stub_generator._untested_version_check(BASE_WORKFLOW_REQUIREMENTS) - installed = { - "grpcio": "1.80.0", - "protobuf": "7.0.1", - "redis": "8.1.0", - "flask": "3.1.3", - } - buffer = io.StringIO() - with mock.patch( - "importlib.metadata.version", side_effect=installed.__getitem__ - ): - with redirect_stdout(buffer): - exec(check, {}) - self.assertEqual( - buffer.getvalue().splitlines(), - [ - "WARNING: protobuf 7.0.1 is newer than CanyonOS has tested (up to 6.x) and may not work." - ], - ) - def _dockerfile(self, workflow, requirements=None): with tempfile.TemporaryDirectory() as tmpdir: project = Path(tmpdir) @@ -615,20 +593,6 @@ def _dockerfile(self, workflow, requirements=None): ) return _read_dockerfile(output_dir) - def test_the_untested_version_check_covers_only_that_images_base(self): - for workflow, base in ( - (False, BASE_AGENT_REQUIREMENTS), - (True, BASE_WORKFLOW_REQUIREMENTS), - ): - with self.subTest(workflow=workflow): - self.assertIn( - f'RUN python -c "{stub_generator._untested_version_check(base)}"', - self._dockerfile(workflow), - ) - self.assertNotIn( - "'flask'", stub_generator._untested_version_check(BASE_AGENT_REQUIREMENTS) - ) - def test_the_proxy_gets_its_own_venv_after_the_app_install(self): for workflow in (False, True): with self.subTest(workflow=workflow): @@ -645,6 +609,61 @@ def test_the_proxy_gets_its_own_venv_after_the_app_install(self): proxy_stage, ) + def test_installs_are_held_below_each_images_tested_majors(self): + for workflow, caps in ( + (False, "'grpcio<2' 'protobuf<7' 'redis<9'"), + (True, "'grpcio<2' 'protobuf<7' 'redis<9' 'flask<4'"), + ): + with self.subTest(workflow=workflow): + dockerfile = self._dockerfile(workflow) + self.assertIn(f"printf '%s\\n' {caps} > /tmp/tested.txt", dockerfile) + self.assertIn( + "uv pip install --system -r requirements.txt " + "--overrides /tmp/overrides.txt -c /tmp/tested.txt", + dockerfile, + ) + + def test_a_package_the_app_asks_newer_for_is_left_uncapped(self): + dockerfile = self._dockerfile(False, requirements=["protobuf>=7"]) + self.assertIn( + "printf '%s\\n' 'grpcio<2' 'redis<9' > /tmp/tested.txt", dockerfile + ) + + def test_requirements_above_the_tested_major_are_reported(self): + self.assertEqual( + stub_generator.too_new_requirements( + [ + "protobuf>=7", + "protobuf==7.1", + "protobuf==7.*", + "protobuf~=7.0", + "flask>4", + "grpcio>=2", + ], + BASE_WORKFLOW_REQUIREMENTS, + ), + [ + ("protobuf>=7", "protobuf<7"), + ("protobuf==7.1", "protobuf<7"), + ("protobuf==7.*", "protobuf<7"), + ("protobuf~=7.0", "protobuf<7"), + ("flask>4", "flask<4"), + ("grpcio>=2", "grpcio<2"), + ], + ) + + def test_requirements_that_still_allow_a_tested_version_are_not_reported(self): + self.assertEqual( + stub_generator.too_new_requirements( + ["protobuf>6.9", "protobuf>=6,<8", "redis", "yfinance>=9"], + BASE_WORKFLOW_REQUIREMENTS, + ), + [], + ) + + def test_an_agent_flask_pin_is_not_a_base_package_to_warn_about(self): + self.assertEqual(stub_generator.too_new_requirements(["flask>=4"]), []) + def test_agents_no_longer_carry_the_proxys_packages(self): for name in ("flask", "requests", "boto3"): with self.subTest(name=name): @@ -654,7 +673,7 @@ def test_agents_no_longer_carry_the_proxys_packages(self): def test_requirements_below_a_floor_are_reported(self): self.assertEqual( - stub_generator.unsupported_requirements( + stub_generator.too_old_requirements( [ "flask==1.9", "flask<2.3", @@ -675,7 +694,7 @@ def test_requirements_below_a_floor_are_reported(self): def test_requirements_that_reach_a_floor_are_not_reported(self): self.assertEqual( - stub_generator.unsupported_requirements( + stub_generator.too_old_requirements( [ "flask~=2.2", "flask>=2", @@ -689,7 +708,7 @@ def test_requirements_that_reach_a_floor_are_not_reported(self): ) def test_an_agent_may_pin_any_flask_now_the_proxy_has_its_own(self): - self.assertEqual(stub_generator.unsupported_requirements(["flask==1.0"]), []) + self.assertEqual(stub_generator.too_old_requirements(["flask==1.0"]), []) def test_the_workflow_context_decides_the_same_way(self): overrides, _ = self._context(["protobuf>=6.32"], workflow=True) From 6c04360dc77282b8f75af20be41bd7344a688d20 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 22 Sep 2026 21:18:19 -0700 Subject: [PATCH 7/7] readability changes --- packages/core/canyonos_core/stub_generator.py | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/core/canyonos_core/stub_generator.py b/packages/core/canyonos_core/stub_generator.py index c77ce330..ee867753 100644 --- a/packages/core/canyonos_core/stub_generator.py +++ b/packages/core/canyonos_core/stub_generator.py @@ -547,11 +547,7 @@ def _copy_files(output_dir, files_to_copy): def _platform_overrides(requirements): - """Force the protobuf floor, intersected with any bound the app itself declares. - - uv replaces a requirement rather than intersecting it, so the app's own - protobuf bound is folded into the override instead of being dropped. - """ + """Defines the range of versions protobuf can take.""" specifier = PROTOBUF_FLOOR.specifier for requirement in requirements: try: @@ -563,8 +559,8 @@ def _platform_overrides(requirements): return [f"{PROTOBUF_FLOOR.name}{specifier}"] -def _caps_below(spec, floor): - """Whether this one specifier allows no version at or above floor.""" +def _below_supported_version(spec, floor): + """Checks if a version specified in an agents requirements list is below the minimum required version CanyonOS requires.""" try: if spec.operator == "==" and spec.version.endswith(".*"): release = Version(spec.version[:-2]).release @@ -599,13 +595,15 @@ def too_old_requirements(requirements, base_requirements=BASE_AGENT_REQUIREMENTS except InvalidRequirement: continue floor = floors.get(parsed.name.lower()) - if floor and any(_caps_below(spec, floor[0]) for spec in parsed.specifier): + if floor and any( + _below_supported_version(spec, floor[0]) for spec in parsed.specifier + ): unsupported.append((requirement, floor[1])) return unsupported -def _forces_at_or_above(spec, limit): - """Whether this one specifier allows only versions at or above limit.""" +def _above_tested_version(spec, limit): + """Checks if a version specified in an agents requirements list is above the newest version CanyonOS has tested.""" try: if spec.operator == "==" and spec.version.endswith(".*"): return Version(spec.version[:-2]) >= limit @@ -628,7 +626,9 @@ def too_new_requirements(requirements, base_requirements=BASE_AGENT_REQUIREMENTS except InvalidRequirement: continue limit = limits.get(parsed.name.lower()) - if limit and any(_forces_at_or_above(spec, limit) for spec in parsed.specifier): + if limit and any( + _above_tested_version(spec, limit) for spec in parsed.specifier + ): too_new.append((requirement, f"{parsed.name.lower()}<{limit}")) return too_new @@ -639,12 +639,8 @@ def _tested_majors(base_requirements): return {n: t for n, t in TESTED_MAJOR_VERSIONS.items() if n in names} -def _dependency_stage(overrides, base_requirements, requirements): - """Render the install stage: app packages held below the tested majors, then the proxy's own venv. - - uv reads overrides and constraints only from files, so the image writes them; - entries are quoted because a bare `>=` or `<` would be a redirect. - """ +def _dockerfile_install_steps(overrides, base_requirements, requirements): + """Writes the Dockerfile steps that install the agent's packages, capped at the versions CanyonOS has tested, plus the LLM proxy's own separate packages.""" forced = " ".join(f"'{override}'" for override in overrides) asked_newer = { Requirement(requirement).name.lower() @@ -802,7 +798,7 @@ def generate_docker( ENV PYTHONUNBUFFERED=1 -{_dependency_stage(overrides, BASE_AGENT_REQUIREMENTS, requirements or [])} +{_dockerfile_install_steps(overrides, BASE_AGENT_REQUIREMENTS, requirements or [])} COPY . . ENV CANYONOS_AGENT_NAME={agent_name} @@ -985,7 +981,7 @@ def mark_ready_when_serving(): ENV PYTHONUNBUFFERED=1 -{_dependency_stage(overrides, BASE_WORKFLOW_REQUIREMENTS, requirements or [])} +{_dockerfile_install_steps(overrides, BASE_WORKFLOW_REQUIREMENTS, requirements or [])} COPY . . EXPOSE 50051