diff --git a/.claude/skills/porting-to-canyonos/validation/runtime.py b/.claude/skills/porting-to-canyonos/validation/runtime.py
index 0be85da..e92d100 100644
--- a/.claude/skills/porting-to-canyonos/validation/runtime.py
+++ b/.claude/skills/porting-to-canyonos/validation/runtime.py
@@ -48,17 +48,8 @@
def _base_requirements():
- agent = [
- "grpcio",
- "grpcio-tools",
- "redis",
- "pyyaml",
- "psutil",
- "ipdb",
- "ipython",
- "boto3",
- ]
- workflow = [*agent, "flask", "sqlalchemy", "psycopg[binary]"]
+ agent = ["grpcio", "protobuf", "redis"]
+ workflow = [*agent, "flask"]
try:
from canyonos_core import stub_generator
except Exception: # noqa: BLE001 - a broken install must not crash validation
diff --git a/README.md b/README.md
index 7da18dd..35b1fa6 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
- 
+
## CanyonOS turns plain Python into a running, distributed workflow — without changing a line of code.
@@ -138,6 +138,15 @@ canyonos test "Hello World!" --json
### 3. Deploy
+
+Supported dependency versions
+
+If your code imports any of these, it must allow at least this version. Older isn't supported, sorry!
+
+`grpcio>=1.76.0` · `protobuf>=6.31.1` · `redis>=3.5`
+
+
+
Deploy the project fully, configured by the config files.
On deploy success, a `POST` endpoint will be returned, in which you can send your workflow queries to.
diff --git a/packages/core/canyonos_core/cli.py b/packages/core/canyonos_core/cli.py
index ce9bd9b..5df22a7 100644
--- a/packages/core/canyonos_core/cli.py
+++ b/packages/core/canyonos_core/cli.py
@@ -19,7 +19,6 @@
from canyonos_core.controller.utils.config_env import load_config
from canyonos_core.controller.utils.env_file import resolve_env_file
from canyonos_core.schema import (
- DependencyPinConflict,
check_project,
declarations_by_name,
render_violation,
@@ -182,33 +181,6 @@ def _normalize_requirements(agent_cfg):
return list(agent_cfg.get("requirements") or [])
-def _check_dependency_pins(manifest):
- """Fail the build when an app pin cannot share a version with a platform pin.
-
- Every service is checked before the first one is built, so a project with
- two bad pins is told about both instead of one per run.
- """
- from canyonos_core.stub_generator import _platform_overrides
-
- violations = []
- for index, service in enumerate(manifest.agents):
- try:
- _platform_overrides(
- getattr(service, "requirements", ()),
- service=index,
- manifest_path=manifest.path,
- lines=getattr(service, "requirement_lines", ()),
- )
- except DependencyPinConflict as conflict:
- violations.extend(conflict.violations)
-
- if violations:
- _reject(
- violations,
- "Dependency pins rejected: %d conflict(s) found; nothing was built.",
- )
-
-
def _docker_platform():
"""Return the target Docker platform for portable runtime images."""
from canyonos_core.stub_generator import target_docker_platform
@@ -370,13 +342,43 @@ def _run_build(config_path):
# so a file that is not YAML at all is rendered as a violation too, and it
# is handed source_root so a service whose code is missing fails here
# rather than being skipped out of a deploy that then reports success.
- manifest = validate_or_exit(config_path, declarations_dir, source_root)
- _check_dependency_pins(manifest)
+ validate_or_exit(config_path, declarations_dir, source_root)
config = _load_config(config_path)
agents = config.get("agents", [])
package_dir = _get_package_dir()
+ from canyonos_core.stub_generator import (
+ BASE_AGENT_REQUIREMENTS,
+ BASE_WORKFLOW_REQUIREMENTS,
+ too_new_requirements,
+ too_old_requirements,
+ )
+
+ too_old = []
+ for agent in agents:
+ requirements = _normalize_requirements(agent)
+ base = (
+ BASE_WORKFLOW_REQUIREMENTS
+ if agent.get("type", "agent") == "workflow"
+ else BASE_AGENT_REQUIREMENTS
+ )
+ too_old += [
+ f"{agent['name']} currently requires {asked}, but CanyonOS only supports {supported}"
+ for asked, supported in too_old_requirements(requirements, base)
+ ]
+ for asked, tested in too_new_requirements(requirements, base):
+ logger.warning(
+ "%s currently requires %s, but CanyonOS has only tested %s; it may not work",
+ agent["name"],
+ asked,
+ tested,
+ )
+ for message in too_old:
+ logger.error("%s", message)
+ if too_old:
+ sys.exit(1)
+
# -------------------------------------------------------------- #
# Step 1: Discover agent YAML files and generate Python stubs #
# -------------------------------------------------------------- #
diff --git a/packages/core/canyonos_core/controller/local_controller.py b/packages/core/canyonos_core/controller/local_controller.py
index 399382c..18fd8b6 100644
--- a/packages/core/canyonos_core/controller/local_controller.py
+++ b/packages/core/canyonos_core/controller/local_controller.py
@@ -57,6 +57,7 @@
import local_controler_pb2
import local_controler_pb2_grpc
+
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@@ -216,8 +217,7 @@ def _start_llm_proxy(self, redis_host, redis_port):
"""
import socket
import subprocess
-
- import requests
+ import urllib.request
# An orphaned proxy on 8081 would answer the /healthz probe below and mask
# one of ours that never bound, so prove the port free before spawning.
@@ -242,9 +242,13 @@ def _start_llm_proxy(self, redis_host, redis_port):
"CANYONOS_REDIS_PORT": str(redis_port),
}
)
+ # The image gives the proxy its own venv so its packages never share versions with the agent's.
+ proxy_python = "/opt/canyonos-proxy/bin/python"
+ if not os.path.exists(proxy_python):
+ proxy_python = sys.executable
try:
proxy_process = subprocess.Popen(
- [sys.executable, "-m", "canyonos_core.llm_proxy"],
+ [proxy_python, "-m", "canyonos_core.llm_proxy"],
env=proxy_env,
)
except Exception as e:
@@ -257,6 +261,7 @@ def _start_llm_proxy(self, redis_host, redis_port):
# Popen only raises if the process can't be spawned -- it returns a healthy
# handle even if the proxy starts and dies immediately, so poll /healthz.
deadline = time.time() + 10
+ last_error = None
while time.time() < deadline:
if proxy_process.poll() is not None:
raise RuntimeError(
@@ -265,15 +270,18 @@ def _start_llm_proxy(self, redis_host, redis_port):
"otherwise fail silently."
)
try:
- if requests.get("http://127.0.0.1:8081/healthz", timeout=0.5).ok:
+ with urllib.request.urlopen(
+ "http://127.0.0.1:8081/healthz", timeout=0.5
+ ):
break
- except requests.exceptions.RequestException:
- pass
+ except OSError as e:
+ last_error = e
time.sleep(0.2)
else:
proxy_process.kill()
raise RuntimeError(
- "LLM proxy did not become healthy on 127.0.0.1:8081 within 10s; "
+ "LLM proxy did not become healthy on 127.0.0.1:8081 within 10s "
+ f"(last health check: {last_error}); "
"agent LLM calls are routed through it unconditionally and would "
"otherwise fail silently."
)
diff --git a/packages/core/canyonos_core/controller/utils/redis_client.py b/packages/core/canyonos_core/controller/utils/redis_client.py
index a9be158..1593c1a 100644
--- a/packages/core/canyonos_core/controller/utils/redis_client.py
+++ b/packages/core/canyonos_core/controller/utils/redis_client.py
@@ -34,7 +34,10 @@ def lock(self, name, timeout):
def expire(self, key, seconds, nx=False):
"""Set a TTL (in seconds) on a key. No-op if the key does not exist."""
- return self.client.expire(key, seconds, nx=nx)
+ # Raw command because redis-py only takes nx= from 4.2.
+ return self.client.execute_command(
+ "EXPIRE", key, seconds, *(["NX"] if nx else [])
+ )
# --- List operations ---
diff --git a/packages/core/canyonos_core/llm_proxy/providers/__init__.py b/packages/core/canyonos_core/llm_proxy/providers/__init__.py
index 1ca3bd1..a32e19f 100644
--- a/packages/core/canyonos_core/llm_proxy/providers/__init__.py
+++ b/packages/core/canyonos_core/llm_proxy/providers/__init__.py
@@ -1,14 +1,21 @@
from __future__ import annotations
from canyonos_core.llm_proxy.providers.anthropic import AnthropicProvider
-from canyonos_core.llm_proxy.providers.bedrock import BedrockProvider
from canyonos_core.llm_proxy.providers.openai import OpenAIProvider
+# boto3 ships only in images whose agent declares it, so Bedrock is registered only when present.
+try:
+ from canyonos_core.llm_proxy.providers.bedrock import BedrockProvider
+except ImportError:
+ BedrockProvider = None
+
def build_registry(cfg):
"""Map the URL prefix -> provider instance."""
- return {
+ registry = {
"openai": OpenAIProvider(cfg),
"anthropic": AnthropicProvider(cfg),
- "bedrock": BedrockProvider(cfg),
}
+ if BedrockProvider is not None:
+ registry["bedrock"] = BedrockProvider(cfg)
+ return registry
diff --git a/packages/core/canyonos_core/stub_generator.py b/packages/core/canyonos_core/stub_generator.py
index 876cdb5..d420c3f 100644
--- a/packages/core/canyonos_core/stub_generator.py
+++ b/packages/core/canyonos_core/stub_generator.py
@@ -13,68 +13,45 @@
import argparse
import ast
import os
-import re
import shutil
-from packaging.requirements import Requirement
-from packaging.utils import canonicalize_name
-from packaging.version import Version
-
-from canyonos_core.schema import (
- DependencyPinConflict,
- SchemaViolation,
- load_agent_declaration,
-)
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.version import InvalidVersion, Version
+
+from canyonos_core.schema import load_agent_declaration
-# Packages every agent container needs regardless of its specific business logic.
-#
-# protobuf and grpcio-tools move together: grpcio-tools carries the only upper
-# bound on protobuf here (1.65.5 capped it below 6.0), and a runtime older than
-# the gencode of any *_pb2.py in the image refuses to load. Transitively
-# installed packages ship gencode 6.x -- googleapis-common-protos, pulled in by
-# the OTLP gRPC exporter, is one -- so a 5.x runtime crashed on import with
-# "gencode 6.33.5 runtime 5.29.6". Neither uv nor pip can reject that pairing,
-# because the constraint lives in the generated module, not in any metadata.
+# Lowest versions the image's own code runs on; an app asking for older fails the install.
BASE_AGENT_REQUIREMENTS = [
- "grpcio==1.83.1",
- "grpcio-tools==1.76.0",
- "protobuf==6.33.5",
- "redis==8.1.0",
- "pyyaml==6.0.3",
- "psutil==7.2.2",
- "boto3==1.43.91",
- "flask==3.1.3",
- "requests==2.34.2",
+ "grpcio>=1.76.0",
+ "protobuf>=6.31.1",
+ "redis>=3.5",
]
-# Workflow containers currently need nothing beyond the base agent requirements
-# (telemetry and session state moved to Redis/OTLP, so no SQL driver is required).
-BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + []
+# Newest major version of each base package CanyonOS is tested on; installs stay below the next major unless the app asks for newer, which only warns.
+TESTED_MAJOR_VERSIONS = {
+ "grpcio": 1,
+ "protobuf": 6,
+ "redis": 8,
+ "flask": 3,
+}
-IMAGE_PYTHON_VERSION = "3.11"
-DEFAULT_DOCKER_PLATFORM = "linux/amd64"
-_MACHINE_BY_DOCKER_ARCH = {"amd64": "x86_64", "arm64": "aarch64"}
-_MARKER_VARIABLES = frozenset(
- {
- "implementation_name",
- "implementation_version",
- "os_name",
- "platform_machine",
- "platform_python_implementation",
- "platform_release",
- "platform_system",
- "platform_version",
- "python_full_version",
- "python_version",
- "sys_platform",
- }
+# deploy.py serves the workflow's HTTP API from the workflow's own process.
+BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask>=2.3.3"]
+
+# The LLM proxy runs from its own venv, so these exact pins never meet the app's.
+# These requirements are not pinned to a specific version because LLM-proxy is completely managed by CanyonOS, with no user code interacting with the internals
+PROXY_REQUIREMENTS = ["flask==3.1.3", "requests==2.34.2", "redis==8.1.0"]
+
+# Only images whose app can import boto3 can call Bedrock, so only they get the proxy's Bedrock route.
+PROXY_BEDROCK_REQUIREMENT = "boto3==1.43.91"
+
+# Every *_pb2.py checks this floor at import, which no package metadata carries,
+# so it is forced past transitive bounds rather than left to the resolver.
+PROTOBUF_FLOOR = Requirement(
+ next(pin for pin in BASE_AGENT_REQUIREMENTS if pin.startswith("protobuf"))
)
-# Packages the image's own code is built against, so an app cannot be left to
-# pick them alone.
-_FORCED_FROM_BASE = ("protobuf", "grpcio", "grpcio-tools", "requests", "boto3")
-PLATFORM_PINS = [
- pin for pin in BASE_AGENT_REQUIREMENTS if pin.split("==")[0] in _FORCED_FROM_BASE
-]
+IMAGE_PYTHON_VERSION = "3.11"
+DEFAULT_DOCKER_PLATFORM = "linux/amd64"
def _build_import_nodes():
@@ -576,132 +553,119 @@ def target_docker_platform():
return os.environ.get("CANYONOS_DOCKER_PLATFORM", DEFAULT_DOCKER_PLATFORM)
-def _image_marker_environment():
- """The marker values the image fixes; the rest are unknown until it runs."""
- environment = {
- "python_version": IMAGE_PYTHON_VERSION,
- "sys_platform": "linux",
- "platform_system": "Linux",
- "os_name": "posix",
- "implementation_name": "cpython",
- "platform_python_implementation": "CPython",
- }
- arch = target_docker_platform().partition("/")[2].partition("/")[0]
- if arch in _MACHINE_BY_DOCKER_ARCH:
- environment["platform_machine"] = _MACHINE_BY_DOCKER_ARCH[arch]
- return environment
-
-
-def _applies_in_image(marker):
- """False only when the marker is known to be false inside the image.
-
- A marker reading a value the image does not fix, such as the Python patch
- version, is checked rather than guessed from the machine running the build.
- """
- environment = _image_marker_environment()
- unquoted = re.sub(r"'[^']*'|\"[^\"]*\"", "", str(marker))
- used = _MARKER_VARIABLES.intersection(re.findall(r"[a-z_]+", unquoted))
- if used - environment.keys():
- return True
- return marker.evaluate(environment)
+def _platform_overrides(requirements):
+ """Defines the range of versions protobuf can take."""
+ specifier = PROTOBUF_FLOOR.specifier
+ for requirement in requirements:
+ try:
+ parsed = Requirement(requirement)
+ except InvalidRequirement:
+ continue
+ if parsed.name.lower() == PROTOBUF_FLOOR.name:
+ specifier &= parsed.specifier
+ return [f"{PROTOBUF_FLOOR.name}{specifier}"]
-def _only_newer_than(spec, pinned):
- """True when `spec` rules `pinned` out only by demanding something newer."""
- if spec.operator not in (">=", ">", "==", "~="):
+def _below_supported_version(spec, floor):
+ """Checks if a version specified in an agents requirements list is below the minimum required version CanyonOS requires."""
+ try:
+ if spec.operator == "==" and spec.version.endswith(".*"):
+ release = Version(spec.version[:-2]).release
+ return (
+ Version(".".join(map(str, (*release[:-1], release[-1] + 1)))) <= floor
+ )
+ version = Version(spec.version)
+ except InvalidVersion:
return False
- bound = Version(spec.version.rstrip(".*"))
- # `>PIN` excludes the pin itself and nothing older, so every version it
- # allows is newer; the other operators need a version past the pin for that.
- return bound >= pinned if spec.operator == ">" else bound > pinned
-
+ if spec.operator in ("==", "==="):
+ return version < floor
+ if spec.operator == "<":
+ return version <= floor
+ if spec.operator == "<=":
+ return version < floor
+ if spec.operator == "~=":
+ release = version.release
+ return Version(".".join(map(str, (*release[:-2], release[-2] + 1)))) <= floor
+ return False
+
+
+def too_old_requirements(requirements, base_requirements=BASE_AGENT_REQUIREMENTS):
+ """Return (requirement, our_floor) for each requirement that only allows versions older than CanyonOS supports."""
+ floors = {}
+ for base in base_requirements:
+ parsed = Requirement(base)
+ floors[parsed.name] = (Version(next(iter(parsed.specifier)).version), base)
+ unsupported = []
+ for requirement in requirements:
+ try:
+ parsed = Requirement(requirement)
+ except InvalidRequirement:
+ continue
+ floor = floors.get(parsed.name.lower())
+ if floor and any(
+ _below_supported_version(spec, floor[0]) for spec in parsed.specifier
+ ):
+ unsupported.append((requirement, floor[1]))
+ return unsupported
-def _platform_overrides(requirements, *, service=None, manifest_path=None, lines=None):
- """Take the higher of each platform pin and what the app asked for.
- uv replaces a requirement rather than intersecting it, so the comparison
- cannot be left to the resolver.
+def _above_tested_version(spec, limit):
+ """Checks if a version specified in an agents requirements list is above the newest version CanyonOS has tested."""
+ try:
+ if spec.operator == "==" and spec.version.endswith(".*"):
+ return Version(spec.version[:-2]) >= limit
+ version = Version(spec.version)
+ except InvalidVersion:
+ return False
+ return spec.operator in ("==", "===", ">=", ">", "~=") and version >= limit
- `service` is the manifest index of the service these requirements belong
- to, and with `manifest_path` and `lines` (each requirement's line) it is
- only there to point a conflict at the line the user has to edit.
- Raises:
- DependencyPinConflict: the app pinned a package *below* the version the
- image's own code is built against. Forcing the platform pin over it
- produced an image that installed cleanly and then failed at import,
- so the build stops here instead.
- """
- declared = {}
- first_lines = {}
- for index, requirement in enumerate(requirements):
- parsed = Requirement(requirement)
- if parsed.marker and not _applies_in_image(parsed.marker):
- continue
- # PEP 503 names: `grpcio_tools`, `Grpcio-Tools` and `grpcio.tools`
- # are all the package pinned as `grpcio-tools`. A package asked for
- # more than once is asked for once with every bound, since only the
- # intersection can be installed -- keeping just the last line made the
- # answer depend on the order they were written in.
- key = canonicalize_name(parsed.name)
- if lines and key not in first_lines:
- first_lines[key] = lines[index]
- if key in declared:
- first_name, specifier = declared[key]
- declared[key] = (first_name, specifier & parsed.specifier)
- else:
- declared[key] = (parsed.name, parsed.specifier)
-
- overrides = []
- conflicts = []
- for pin in PLATFORM_PINS:
- name, pinned = pin.split("==")
- pinned_version = Version(pinned)
- asked = declared.get(canonicalize_name(name))
- if asked is None or asked[1].contains(pinned_version):
- overrides.append(pin)
+def too_new_requirements(requirements, base_requirements=BASE_AGENT_REQUIREMENTS):
+ """Return (requirement, tested_limit) for each requirement that only allows versions newer than CanyonOS has tested."""
+ limits = {
+ name: Version(str(major + 1))
+ for name, major in _tested_majors(base_requirements).items()
+ }
+ too_new = []
+ for requirement in requirements:
+ try:
+ parsed = Requirement(requirement)
+ except InvalidRequirement:
continue
- asked_name, specifier = asked
- wanted = f"{asked_name}{specifier}"
- # Newer only if a lower bound above the pin rules it out and nothing
- # else does but an exclusion; one upper bound below it and nothing
- # newer can satisfy both.
- ruling = [spec for spec in specifier if not spec.contains(pinned_version)]
- if any(_only_newer_than(spec, pinned_version) for spec in ruling) and all(
- _only_newer_than(spec, pinned_version) or spec.operator == "!="
- for spec in ruling
+ limit = limits.get(parsed.name.lower())
+ if limit and any(
+ _above_tested_version(spec, limit) for spec in parsed.specifier
):
- overrides.append(wanted)
- print(f" Note: '{wanted}' outranks the platform pin {pin}")
- else:
- overrides.append(pin)
- field = (
- "requirements" if service is None else f"agents[{service}].requirements"
- )
- conflicts.append(
- SchemaViolation(
- manifest_path or "",
- first_lines.get(canonicalize_name(name), 0),
- field,
- f"'{wanted}' conflicts with the platform pin {pin}, which the "
- f"agent image is built against: relax the bound or pin "
- f"{name} at or above {pinned}",
- )
- )
- if conflicts:
- raise DependencyPinConflict(conflicts)
- return overrides
+ too_new.append((requirement, f"{parsed.name.lower()}<{limit}"))
+ return too_new
+
+
+def _tested_majors(base_requirements):
+ """TESTED_MAJOR_VERSIONS narrowed to the packages this image's base list installs."""
+ names = {Requirement(r).name for r in base_requirements}
+ return {n: t for n, t in TESTED_MAJOR_VERSIONS.items() if n in names}
-def _dependency_stage(overrides):
- """Render the install stage. uv reads overrides from a file and takes no
- inline form, so the image writes one; the entries are quoted because a bare
- `>=` would be a redirect."""
+def _dockerfile_install_steps(overrides, base_requirements, requirements):
+ """Writes the Dockerfile steps that install the agent's packages, capped at the versions CanyonOS has tested, plus the LLM proxy's own separate packages."""
forced = " ".join(f"'{override}'" for override in overrides)
+ asked_newer = {
+ Requirement(requirement).name.lower()
+ for requirement, _ in too_new_requirements(requirements, base_requirements)
+ }
+ caps = " ".join(
+ f"'{name}<{major + 1}'"
+ for name, major in _tested_majors(base_requirements).items()
+ if name not in asked_newer
+ )
return f"""COPY requirements.txt .
RUN --mount=type=cache,target=/root/.cache/uv printf '%s\\n' {forced} > /tmp/overrides.txt \\
- && uv pip install --system -r requirements.txt --overrides /tmp/overrides.txt
+ && printf '%s\\n' {caps} > /tmp/tested.txt \\
+ && uv pip install --system -r requirements.txt --overrides /tmp/overrides.txt -c /tmp/tested.txt
RUN uv pip check --system || echo "NOTE: CanyonOS forces {forced}; an incompatibility above naming one of those is a bound it could not share with the app."
+RUN --mount=type=cache,target=/root/.cache/uv uv venv /opt/canyonos-proxy \\
+ && uv pip install --python /opt/canyonos-proxy/bin/python {" ".join(PROXY_REQUIREMENTS)} \\
+ && if python -c "import boto3" 2>/dev/null; then uv pip install --python /opt/canyonos-proxy/bin/python {PROXY_BEDROCK_REQUIREMENT}; fi
"""
@@ -838,7 +802,7 @@ def generate_docker(
ENV PYTHONUNBUFFERED=1
-{_dependency_stage(overrides)}
+{_dockerfile_install_steps(overrides, BASE_AGENT_REQUIREMENTS, requirements or [])}
COPY . .
ENV CANYONOS_AGENT_NAME={agent_name}
@@ -1021,7 +985,7 @@ def mark_ready_when_serving():
ENV PYTHONUNBUFFERED=1
-{_dependency_stage(overrides)}
+{_dockerfile_install_steps(overrides, BASE_WORKFLOW_REQUIREMENTS, requirements or [])}
COPY . .
EXPOSE 50051
diff --git a/packages/core/tests/test_cli.py b/packages/core/tests/test_cli.py
index a6433c1..b9d86f4 100644
--- a/packages/core/tests/test_cli.py
+++ b/packages/core/tests/test_cli.py
@@ -269,6 +269,51 @@ def _write_agent_and_workflow_config(self, project_dir):
)
return agent_yaml
+ def test_build_stops_on_a_workflow_requirement_below_the_supported_floor(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ project_dir = Path(tmpdir)
+ agent_yaml = self._write_agent_and_workflow_config(project_dir)
+ config_path = project_dir / "config" / "global_controller.yaml"
+ config = yaml.safe_load(config_path.read_text())
+ config["agents"][0]["requirements"] = ["flask==1.9"]
+ config["agents"][1]["requirements"] = ["flask==1.9"]
+ config_path.write_text(yaml.safe_dump(config))
+
+ with (
+ self.assertRaises(SystemExit),
+ self.assertLogs("canyonos_core", level="ERROR") as logs,
+ ):
+ self._run_build(project_dir, [str(agent_yaml)], buildx_available=True)
+
+ self.assertEqual(
+ logs.output,
+ [
+ "ERROR:canyonos_core:Workflow currently requires flask==1.9, "
+ "but CanyonOS only supports flask>=2.3.3"
+ ],
+ )
+
+ def test_build_warns_on_a_requirement_above_the_tested_major_and_continues(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ project_dir = Path(tmpdir)
+ agent_yaml = self._write_agent_and_workflow_config(project_dir)
+ config_path = project_dir / "config" / "global_controller.yaml"
+ config = yaml.safe_load(config_path.read_text())
+ config["agents"][0]["requirements"] = ["protobuf>=7"]
+ config_path.write_text(yaml.safe_dump(config))
+
+ with self.assertLogs("canyonos_core", level="WARNING") as logs:
+ _, generate_docker, _ = self._run_build(
+ project_dir, [str(agent_yaml)], buildx_available=True
+ )
+
+ self.assertIn(
+ "WARNING:canyonos_core:ExampleAgent currently requires protobuf>=7, "
+ "but CanyonOS has only tested protobuf<7; it may not work",
+ logs.output,
+ )
+ generate_docker.assert_called_once()
+
def test_build_falls_back_to_sequential_docker_build_without_buildx(self):
with tempfile.TemporaryDirectory() as tmpdir:
project_dir = Path(tmpdir)
@@ -592,9 +637,9 @@ def test_build_rejects_a_dependency_pin_the_platform_cannot_meet(self):
)
# Both services are reported, so two bad pins take one run to find.
- self.assertIn("agents[0].requirements", log.output[0])
+ self.assertIn("ExampleAgent", log.output[0])
self.assertIn("protobuf<5", log.output[0])
- self.assertIn("agents[1].requirements", log.output[1])
+ self.assertIn("OtherAgent", log.output[1])
class BuildStopsBeforeGeneratingAnythingTests(unittest.TestCase):
diff --git a/packages/core/tests/test_local_controller_proxy_start.py b/packages/core/tests/test_local_controller_proxy_start.py
new file mode 100644
index 0000000..91a1ede
--- /dev/null
+++ b/packages/core/tests/test_local_controller_proxy_start.py
@@ -0,0 +1,41 @@
+import unittest
+import urllib.error
+from unittest.mock import MagicMock, patch
+
+from canyonos_core.controller.local_controller import LocalController
+
+
+class StartLlmProxyTests(unittest.TestCase):
+ def _start_with_health_check_failing(self, error):
+ proxy_process = MagicMock()
+ proxy_process.poll.return_value = None
+ fake_time = MagicMock()
+ fake_time.time.side_effect = [0, 0, 11]
+ with (
+ patch("socket.socket"),
+ patch("subprocess.Popen", return_value=proxy_process),
+ patch("urllib.request.urlopen", side_effect=error),
+ patch("canyonos_core.controller.local_controller.time", fake_time),
+ self.assertRaises(RuntimeError) as raised,
+ ):
+ LocalController._start_llm_proxy(MagicMock(), "localhost", 6379)
+ proxy_process.kill.assert_called_once()
+ return str(raised.exception)
+
+ def test_the_timeout_names_the_last_health_check_failure(self):
+ message = self._start_with_health_check_failing(
+ urllib.error.URLError(ConnectionRefusedError(111, "Connection refused"))
+ )
+ self.assertIn("did not become healthy on 127.0.0.1:8081 within 10s", message)
+ self.assertIn(
+ "(last health check: )",
+ message,
+ )
+
+ def test_a_timed_out_health_check_is_named_too(self):
+ message = self._start_with_health_check_failing(TimeoutError("timed out"))
+ self.assertIn("(last health check: timed out)", message)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/packages/core/tests/test_stub_generator.py b/packages/core/tests/test_stub_generator.py
index 40fc7b6..3255d80 100644
--- a/packages/core/tests/test_stub_generator.py
+++ b/packages/core/tests/test_stub_generator.py
@@ -8,17 +8,16 @@
from pathlib import Path
import yaml
-from packaging.version import Version
+from packaging.requirements import Requirement
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from canyonos_core import stub_generator
-from canyonos_core.schema import DependencyPinConflict, render_violation
from canyonos_core.stub_generator import (
BASE_AGENT_REQUIREMENTS,
BASE_WORKFLOW_REQUIREMENTS,
- PLATFORM_PINS,
- _platform_overrides,
+ PROTOBUF_FLOOR,
+ TESTED_MAJOR_VERSIONS,
_stub_destination,
_sweep_project_files,
generate_docker,
@@ -55,15 +54,9 @@ def test_base_only_when_requirements_omitted(self):
self.assertEqual(
requirements,
[
- "grpcio==1.83.1",
- "grpcio-tools==1.76.0",
- "protobuf==6.33.5",
- "redis==8.1.0",
- "pyyaml==6.0.3",
- "psutil==7.2.2",
- "boto3==1.43.91",
- "flask==3.1.3",
- "requests==2.34.2",
+ "grpcio>=1.76.0",
+ "protobuf>=6.31.1",
+ "redis>=3.5",
],
)
self.assertNotIn("yfinance", requirements)
@@ -486,7 +479,7 @@ def test_stub_lands_both_flat_and_at_its_entrypoint_path(self):
class PlatformPinTests(unittest.TestCase):
- """Each forced package resolves to the higher of our pin and the app's ask."""
+ """Only protobuf is forced, intersected with whatever bound the app declares."""
def _context(self, requirements, workflow=False):
with tempfile.TemporaryDirectory() as tmpdir:
@@ -524,199 +517,199 @@ def _context(self, requirements, workflow=False):
]
return [entry.strip("'") for entry in written.split()], notes
- def test_every_platform_pin_is_exact(self):
- for pin in PLATFORM_PINS:
- with self.subTest(pin=pin):
- self.assertRegex(pin, r"^[a-z0-9-]+==[0-9][0-9a-z.]*$")
-
- def test_pins_come_from_the_base_requirements(self):
- forced = {pin.split("==")[0] for pin in PLATFORM_PINS}
- self.assertEqual(forced, set(stub_generator._FORCED_FROM_BASE))
- for pin in PLATFORM_PINS:
- with self.subTest(pin=pin):
- self.assertIn(pin, BASE_AGENT_REQUIREMENTS)
-
- def test_grpcio_tools_is_forced_wherever_protobuf_is(self):
- # protoc stamps its own generation into the *_pb2.py it writes.
- forced = {pin.split("==")[0] for pin in PLATFORM_PINS}
- if "protobuf" in forced:
- self.assertIn("grpcio-tools", forced)
-
- def test_the_forced_protobuf_satisfies_the_grpcio_tools_bound(self):
- # grpcio-tools carries the only upper bound on protobuf in the base set,
- # so the two cannot be bumped independently: 1.65.5 required
- # protobuf<6.0, which held the runtime below the gencode 6.x that
- # transitively installed *_pb2.py modules are built with. 1.76.0
- # requires >=6.31.1.
- pins = {pin.split("==")[0]: pin.split("==")[1] for pin in PLATFORM_PINS}
- self.assertGreaterEqual(Version(pins["protobuf"]), Version("6.31.1"))
-
- def test_the_pin_holds_and_stays_quiet_when_nothing_newer_is_asked(self):
+ def test_base_requirements_are_ranges_not_exact_pins(self):
+ for requirement in BASE_AGENT_REQUIREMENTS:
+ with self.subTest(requirement=requirement):
+ self.assertNotIn("==", requirement)
+
+ def test_the_protobuf_floor_loads_host_compiled_gencode(self):
+ # The host's grpcio-tools 1.76.0 emits *_pb2.py that need protobuf>=6.31.1 at runtime.
+ self.assertIn("6.31.1", PROTOBUF_FLOOR.specifier)
+ self.assertNotIn("6.31.0", PROTOBUF_FLOOR.specifier)
+
+ def test_only_protobuf_is_forced(self):
for requirements in (
[],
- ["protobuf>=5.29.0"],
- ["protobuf==6.33.5"],
["streamlit==1.31.1"],
+ ["requests==2.28.0", "flask==2.3.3", "grpcio==1.80.0"],
):
with self.subTest(requirements=requirements):
overrides, notes = self._context(requirements)
- self.assertEqual(overrides, list(PLATFORM_PINS))
+ self.assertEqual(overrides, [str(PROTOBUF_FLOOR)])
self.assertEqual(notes, [])
- def test_an_app_asking_for_newer_wins(self):
- overrides, notes = self._context(["protobuf>=7"])
- self.assertIn("protobuf>=7", overrides)
- self.assertNotIn("protobuf==6.33.5", overrides)
- self.assertEqual(
- notes, ["Note: 'protobuf>=7' outranks the platform pin protobuf==6.33.5"]
- )
+ def test_an_app_protobuf_bound_is_intersected_with_the_floor(self):
+ overrides, _ = self._context(["protobuf>=6.32"])
+ self.assertEqual(len(overrides), 1)
+ forced = Requirement(overrides[0]).specifier
+ self.assertNotIn("6.31.5", forced)
+ self.assertIn("6.33.5", forced)
- def test_an_app_asking_for_older_fails_the_build(self):
- # Forcing the platform pin over the app's bound used to produce an image
- # that installed cleanly and then failed at import, so it is fatal now.
- with self.assertRaises(DependencyPinConflict) as raised:
- self._context(["protobuf<5"])
-
- (violation,) = raised.exception.violations
- self.assertEqual(violation.field, "requirements")
- self.assertIn("protobuf<5", violation.message)
- self.assertIn("protobuf==6.33.5", violation.message)
- self.assertIn("at or above 6.33.5", violation.message)
-
- def test_the_conflict_points_at_the_manifest_entry_to_edit(self):
- with self.assertRaises(DependencyPinConflict) as raised:
- _platform_overrides(
- ["boto3<1"],
- service=2,
- manifest_path="config/global_controller.yaml",
- )
+ def test_other_base_packages_are_left_to_the_resolver(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workflow_file = _write(Path(tmpdir) / "workflow.py", "print('ok')\n")
+ output_dir = os.path.join(tmpdir, "out")
+ with redirect_stdout(io.StringIO()):
+ generate_workflow_docker(
+ str(workflow_file),
+ [],
+ output_dir=output_dir,
+ requirements=["flask==2.3.3"],
+ )
+ requirements = _read_requirements(output_dir)
- (violation,) = raised.exception.violations
- rendered = render_violation(violation)
- self.assertTrue(rendered.startswith("config/global_controller.yaml: "))
- self.assertIn("agents[2].requirements", rendered)
- self.assertNotIn("\n", rendered)
+ self.assertIn("flask>=2.3.3", requirements)
+ self.assertIn("flask==2.3.3", requirements)
- def test_a_conflict_is_caught_however_the_package_name_is_spelled(self):
- # pip treats these as one package; matching on .lower() alone missed
- # the underscore and forced the pin over the app's bound instead.
- for requirement in ("grpcio_tools<1", "Grpcio-Tools<1", "grpcio.tools<1"):
+ def test_base_requirements_carry_no_upper_bound(self):
+ for requirement in BASE_WORKFLOW_REQUIREMENTS:
with self.subTest(requirement=requirement):
- with self.assertRaises(DependencyPinConflict) as raised:
- _platform_overrides([requirement], service=0)
- (violation,) = raised.exception.violations
- self.assertIn("grpcio-tools==1.76.0", violation.message)
+ self.assertNotIn("<", requirement)
- def test_a_newer_ask_wins_however_the_package_name_is_spelled(self):
- overrides = _platform_overrides(["grpcio_tools>=2"])
+ def test_every_base_package_has_a_tested_major_version(self):
+ names = {Requirement(r).name for r in BASE_WORKFLOW_REQUIREMENTS}
+ self.assertEqual(names, set(TESTED_MAJOR_VERSIONS))
- self.assertIn("grpcio_tools>=2", overrides)
- self.assertNotIn("grpcio-tools==1.76.0", overrides)
+ def _dockerfile(self, workflow, requirements=None):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ project = Path(tmpdir)
+ output_dir = os.path.join(tmpdir, "out")
+ with redirect_stdout(io.StringIO()):
+ if workflow:
+ wf = _write(project / "workflow.py", "print('ok')\n")
+ generate_workflow_docker(
+ str(wf), [], output_dir=output_dir, requirements=requirements
+ )
+ else:
+ yaml_path = project / "ExampleAgent.yaml"
+ yaml_path.write_text(
+ yaml.safe_dump({"agent": {"name": "ExampleAgent"}})
+ )
+ agent = _write(project / "agent.py", "print('ok')\n")
+ generate_docker(
+ str(yaml_path),
+ str(agent),
+ output_dir=output_dir,
+ requirements=requirements,
+ )
+ return _read_dockerfile(output_dir)
- def test_a_package_asked_for_twice_conflicts_whatever_the_order(self):
- # Only the last line used to count, so `protobuf>=7` written second
- # hid the `<5` bound and the build went ahead.
- messages = []
- for requirements in (
- ["protobuf<5", "protobuf>=7"],
- ["protobuf>=7", "protobuf<5"],
+ def test_the_proxy_gets_its_own_venv_after_the_app_install(self):
+ for workflow in (False, True):
+ with self.subTest(workflow=workflow):
+ dockerfile = self._dockerfile(workflow)
+ app_install = dockerfile.index("uv pip install --system")
+ proxy_install = dockerfile.index("uv venv /opt/canyonos-proxy")
+ self.assertLess(app_install, proxy_install)
+ proxy_stage = dockerfile[proxy_install:]
+ for pin in stub_generator.PROXY_REQUIREMENTS:
+ self.assertIn(pin, proxy_stage)
+ self.assertIn(
+ 'if python -c "import boto3" 2>/dev/null; then uv pip install '
+ f"--python /opt/canyonos-proxy/bin/python {stub_generator.PROXY_BEDROCK_REQUIREMENT}; fi",
+ proxy_stage,
+ )
+
+ def test_installs_are_held_below_each_images_tested_majors(self):
+ for workflow, caps in (
+ (False, "'grpcio<2' 'protobuf<7' 'redis<9'"),
+ (True, "'grpcio<2' 'protobuf<7' 'redis<9' 'flask<4'"),
):
- with self.subTest(requirements=requirements):
- with self.assertRaises(DependencyPinConflict) as raised:
- _platform_overrides(requirements, service=0)
- (violation,) = raised.exception.violations
- messages.append(violation.message)
+ with self.subTest(workflow=workflow):
+ dockerfile = self._dockerfile(workflow)
+ self.assertIn(f"printf '%s\\n' {caps} > /tmp/tested.txt", dockerfile)
+ self.assertIn(
+ "uv pip install --system -r requirements.txt "
+ "--overrides /tmp/overrides.txt -c /tmp/tested.txt",
+ dockerfile,
+ )
- self.assertEqual(messages[0], messages[1])
- self.assertIn("'protobuf<5,>=7'", messages[0])
+ def test_a_package_the_app_asks_newer_for_is_left_uncapped(self):
+ dockerfile = self._dockerfile(False, requirements=["protobuf>=7"])
+ self.assertIn(
+ "printf '%s\\n' 'grpcio<2' 'redis<9' > /tmp/tested.txt", dockerfile
+ )
- def test_strictly_greater_than_the_pin_is_a_newer_ask(self):
- # Every version `>6.33.5` allows is newer than the pin, but it used to
- # be reported as a conflict because its bound was not past the pin.
- overrides, notes = self._context(["protobuf>6.33.5"])
+ def test_requirements_above_the_tested_major_are_reported(self):
+ self.assertEqual(
+ stub_generator.too_new_requirements(
+ [
+ "protobuf>=7",
+ "protobuf==7.1",
+ "protobuf==7.*",
+ "protobuf~=7.0",
+ "flask>4",
+ "grpcio>=2",
+ ],
+ BASE_WORKFLOW_REQUIREMENTS,
+ ),
+ [
+ ("protobuf>=7", "protobuf<7"),
+ ("protobuf==7.1", "protobuf<7"),
+ ("protobuf==7.*", "protobuf<7"),
+ ("protobuf~=7.0", "protobuf<7"),
+ ("flask>4", "flask<4"),
+ ("grpcio>=2", "grpcio<2"),
+ ],
+ )
- self.assertIn("protobuf>6.33.5", overrides)
- self.assertNotIn("protobuf==6.33.5", overrides)
+ def test_requirements_that_still_allow_a_tested_version_are_not_reported(self):
self.assertEqual(
- notes,
- ["Note: 'protobuf>6.33.5' outranks the platform pin protobuf==6.33.5"],
+ stub_generator.too_new_requirements(
+ ["protobuf>6.9", "protobuf>=6,<8", "redis", "yfinance>=9"],
+ BASE_WORKFLOW_REQUIREMENTS,
+ ),
+ [],
)
- def test_at_or_equal_to_the_pin_keeps_the_pin_quietly(self):
- for requirement in ("protobuf>=6.33.5", "protobuf==6.33.5"):
- with self.subTest(requirement=requirement):
- overrides, notes = self._context([requirement])
- self.assertEqual(overrides, list(PLATFORM_PINS))
- self.assertEqual(notes, [])
+ def test_an_agent_flask_pin_is_not_a_base_package_to_warn_about(self):
+ self.assertEqual(stub_generator.too_new_requirements(["flask>=4"]), [])
- def test_an_exclusion_beside_a_newer_bound_is_still_a_newer_ask(self):
- for requirement in ("protobuf>=7,!=6.33.5", "requests>=3,!=2.34.2"):
- with self.subTest(requirement=requirement):
- with redirect_stdout(io.StringIO()):
- overrides = _platform_overrides([requirement])
- name = requirement.split(">=")[0]
+ def test_agents_no_longer_carry_the_proxys_packages(self):
+ for name in ("flask", "requests", "boto3"):
+ with self.subTest(name=name):
self.assertFalse(
- [pin for pin in overrides if pin.startswith(f"{name}==")]
+ any(Requirement(r).name == name for r in BASE_AGENT_REQUIREMENTS)
)
- def test_excluding_the_pin_alone_is_still_a_conflict(self):
- with self.assertRaises(DependencyPinConflict):
- _platform_overrides(["protobuf!=6.33.5"], service=0)
-
- def test_a_requirement_whose_marker_is_false_in_the_image_is_ignored(self):
- overrides = _platform_overrides(
+ def test_requirements_below_a_floor_are_reported(self):
+ self.assertEqual(
+ stub_generator.too_old_requirements(
+ [
+ "flask==1.9",
+ "flask<2.3",
+ "flask~=2.2.0",
+ "flask==2.2.*",
+ "protobuf<5",
+ ],
+ BASE_WORKFLOW_REQUIREMENTS,
+ ),
[
- 'grpcio<0.1; sys_platform == "win32"',
- "grpcio>=1.60; python_version >= '3.8'",
- "grpcio<1.50; python_version < '3.8'",
- ]
+ ("flask==1.9", "flask>=2.3.3"),
+ ("flask<2.3", "flask>=2.3.3"),
+ ("flask~=2.2.0", "flask>=2.3.3"),
+ ("flask==2.2.*", "flask>=2.3.3"),
+ ("protobuf<5", "protobuf>=6.31.1"),
+ ],
)
- self.assertIn("grpcio==1.83.1", overrides)
-
- def test_a_requirement_whose_marker_is_true_in_the_image_is_checked(self):
- with self.assertRaises(DependencyPinConflict):
- _platform_overrides(['grpcio<0.1; sys_platform == "linux"'], service=0)
-
- def test_a_marker_is_evaluated_for_the_image_architecture(self):
- requirement = 'grpcio<0.1; platform_machine == "aarch64"'
- with unittest.mock.patch.dict(
- os.environ, {"CANYONOS_DOCKER_PLATFORM": "linux/amd64"}
- ):
- self.assertIn("grpcio==1.83.1", _platform_overrides([requirement]))
- with unittest.mock.patch.dict(
- os.environ, {"CANYONOS_DOCKER_PLATFORM": "linux/arm64"}
- ):
- with self.assertRaises(DependencyPinConflict):
- _platform_overrides([requirement], service=0)
-
- def test_a_marker_on_a_value_the_image_does_not_fix_is_still_checked(self):
- with self.assertRaises(DependencyPinConflict):
- _platform_overrides(
- ['grpcio<0.1; python_full_version >= "3.11.99"'], service=0
- )
-
- def test_a_conflict_points_at_the_requirements_line(self):
- with self.assertRaises(DependencyPinConflict) as raised:
- _platform_overrides(
- ["requests", "grpcio_tools<1", "grpcio-tools<0.5"],
- service=0,
- manifest_path="config/global_controller.yaml",
- lines=[7, 8, 9],
- )
-
- (violation,) = raised.exception.violations
- self.assertEqual(violation.line, 8)
- self.assertTrue(
- render_violation(violation).startswith("config/global_controller.yaml:8: ")
+ def test_requirements_that_reach_a_floor_are_not_reported(self):
+ self.assertEqual(
+ stub_generator.too_old_requirements(
+ [
+ "flask~=2.2",
+ "flask>=2",
+ "flask!=2.3.3",
+ "flask==2.3.3",
+ "yfinance==0.1",
+ ],
+ BASE_WORKFLOW_REQUIREMENTS,
+ ),
+ [],
)
- def test_two_newer_asks_for_one_package_still_win(self):
- with redirect_stdout(io.StringIO()):
- overrides = _platform_overrides(["protobuf>=7", "Protobuf>=7.1"])
-
- self.assertIn("protobuf>=7,>=7.1", overrides)
- self.assertNotIn("protobuf==6.33.5", overrides)
+ def test_an_agent_may_pin_any_flask_now_the_proxy_has_its_own(self):
+ self.assertEqual(stub_generator.too_old_requirements(["flask==1.0"]), [])
def test_a_repeated_package_is_still_written_line_for_line(self):
# Combining the bounds is only for the comparison; requirements.txt
@@ -738,14 +731,9 @@ def test_a_repeated_package_is_still_written_line_for_line(self):
self.assertEqual(requirements[-2:], ["protobuf>=7", "Protobuf>=7.1"])
- def test_the_workflow_context_fails_on_a_conflict_too(self):
- with self.assertRaises(DependencyPinConflict):
- self._context(["protobuf<5"], workflow=True)
-
def test_the_workflow_context_decides_the_same_way(self):
- overrides, notes = self._context(["protobuf>=7"], workflow=True)
- self.assertIn("protobuf>=7", overrides)
- self.assertEqual(len(notes), 1)
+ overrides, _ = self._context(["protobuf>=6.32"], workflow=True)
+ self.assertEqual(overrides, self._context(["protobuf>=6.32"])[0])
def test_override_entries_are_quoted_for_the_shell(self):
# An unquoted `protobuf>=7` would be a redirect, not an argument.
@@ -760,11 +748,11 @@ def test_override_entries_are_quoted_for_the_shell(self):
str(yaml_path),
str(agent_file),
output_dir=output_dir,
- requirements=["protobuf>=7"],
+ requirements=["protobuf>=6.32"],
)
dockerfile = _read_dockerfile(output_dir)
- self.assertIn("'protobuf>=7'", dockerfile)
+ self.assertIn("'protobuf>=6.31.1,>=6.32'", dockerfile)
def test_both_dockerfiles_install_with_the_overrides_and_report(self):
for workflow in (False, True):
@@ -790,8 +778,7 @@ def test_both_dockerfiles_install_with_the_overrides_and_report(self):
install = dockerfile.split("RUN uv pip check")[0]
self.assertIn("--overrides /tmp/overrides.txt", install)
self.assertIn("uv pip check --system", dockerfile)
- for pin in PLATFORM_PINS:
- self.assertIn(pin, dockerfile.split("NOTE:")[1])
+ self.assertIn(str(PROTOBUF_FLOOR), dockerfile.split("NOTE:")[1])
if __name__ == "__main__":