From 4d28f95d8bb56f70fb2507642278975948ad448f Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 17:26:35 -0700 Subject: [PATCH 01/31] includes all the files when ventis build --- ventis/cli.py | 2 ++ ventis/stub_generator.py | 73 ++++++++++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 3aceb18..9ffc149 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -274,6 +274,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), + project_dir=project_dir, ) else: @@ -316,6 +317,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, + project_dir=project_dir, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 8c956ef..a571ccc 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -263,8 +263,36 @@ def _format_source(source): return "\n".join(formatted) + "\n" +# Directories ventis build itself generates inside a project -- never swept. +_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} + + +def _sweep_py_files(project_dir): + """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" + swept = [] + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + for fname in files: + if fname.endswith(".py"): + abs_src = os.path.join(root, fname) + rel_dst = os.path.relpath(abs_src, project_dir) + swept.append((abs_src, rel_dst)) + return swept + + +def _stub_destination(stub_file, project_dir): + """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" + basename = os.path.basename(stub_file) + return os.path.join("agents", basename) if project_dir else basename + + def generate_docker( - yaml_path, agent_file, output_dir=None, grpc_stubs_dir=None, stub_files=None + yaml_path, + agent_file, + output_dir=None, + grpc_stubs_dir=None, + stub_files=None, + project_dir=None, ): """ Generate a minimal Docker build context for an agent. @@ -278,6 +306,7 @@ def generate_docker( output_dir: Optional output directory (default: docker_container//). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -301,8 +330,13 @@ def generate_docker( with open(os.path.join(output_dir, "requirements.txt"), "w") as f: f.write(requirements) + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + # Copy general agent files - files_to_copy = [ + files_to_copy += [ # (source_path, destination_filename) (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -322,13 +356,13 @@ def generate_docker( (os.path.join(script_dir, "llm", "bedrock.py"), "bedrock.py"), ] - # Copy provided agent stubs + # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), os.path.basename(stub_file)) + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) ) - + files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) # Copy gRPC generated stubs if they exist @@ -339,7 +373,9 @@ def generate_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") @@ -377,7 +413,12 @@ def generate_docker( def generate_workflow_docker( - workflow_file, stub_files, output_dir=None, grpc_stubs_dir=None, api_port=8080 + workflow_file, + stub_files, + output_dir=None, + grpc_stubs_dir=None, + api_port=8080, + project_dir=None, ): """ Generate a Docker build context for a workflow. @@ -391,6 +432,7 @@ def generate_workflow_docker( stub_files: List of stub file paths to include. output_dir: Optional output directory (default: docker_container/Workflow/). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -417,7 +459,12 @@ def generate_workflow_docker( # ---- Copy source files into the build context ------------------------ workflow_basename = os.path.basename(workflow_file) - files_to_copy = [ + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + + files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -437,9 +484,11 @@ def generate_workflow_docker( ], ] - # Copy stub files + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) + files_to_copy.append( + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -449,7 +498,9 @@ def generate_workflow_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") From 23e3928d01bc5996fd1cd4e086d45212c283de0e Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 18:23:51 -0700 Subject: [PATCH 02/31] fixed some bugs --- ventis/cli.py | 19 ++++++++++++++ ventis/stub_generator.py | 54 +++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 9ffc149..cb9ee0a 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -220,6 +220,23 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) + # Map each stub's basename to its agent's declared entrypoint, so a stub + # overwrites the exact real file it replaces instead of guessing its path. + stub_entrypoints = {} + for agent_cfg in agents: + entrypoint = agent_cfg.get("entrypoint") + if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: + continue + for yaml_path in yaml_files: + import yaml + + with open(yaml_path) as f: + ydata = yaml.safe_load(f) + if ydata.get("agent", {}).get("name") == agent_cfg["name"]: + base_name = os.path.splitext(os.path.basename(yaml_path))[0] + stub_entrypoints[f"{base_name}.py"] = entrypoint + break + # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -275,6 +292,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) else: @@ -318,6 +336,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a571ccc..a56080d 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -271,7 +271,12 @@ def _sweep_py_files(project_dir): """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" swept = [] for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") + and not (root == project_dir and d in _GENERATED_DIRS) + ] for fname in files: if fname.endswith(".py"): abs_src = os.path.join(root, fname) @@ -280,10 +285,15 @@ def _sweep_py_files(project_dir): return swept -def _stub_destination(stub_file, project_dir): - """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" +def _stub_destination(stub_file, stub_entrypoints): + """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" basename = os.path.basename(stub_file) - return os.path.join("agents", basename) if project_dir else basename + entrypoint = stub_entrypoints.get(basename) + if entrypoint: + normalized = entrypoint.replace("\\", "/") + if not normalized.startswith("/") and ".." not in normalized.split("/"): + return entrypoint + return basename def generate_docker( @@ -293,6 +303,7 @@ def generate_docker( grpc_stubs_dir=None, stub_files=None, project_dir=None, + stub_entrypoints=None, ): """ Generate a minimal Docker build context for an agent. @@ -301,12 +312,13 @@ def generate_docker( source files needed to run the agent with its own local controller. Args: - yaml_path: Path to the YAML agent definition. - agent_file: Path to the original Python agent implementation. - output_dir: Optional output directory (default: docker_container//). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - stub_files: Optional list of agent stub files to copy into the context. - project_dir: Optional project root to sweep for extra .py helper files. + yaml_path: Path to the YAML agent definition. + agent_file: Path to the original Python agent implementation. + output_dir: Optional output directory (default: docker_container//). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -360,7 +372,10 @@ def generate_docker( if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) @@ -419,6 +434,7 @@ def generate_workflow_docker( grpc_stubs_dir=None, api_port=8080, project_dir=None, + stub_entrypoints=None, ): """ Generate a Docker build context for a workflow. @@ -428,11 +444,12 @@ def generate_workflow_docker( with its own local controller. Args: - workflow_file: Path to the workflow Python file. - stub_files: List of stub file paths to include. - output_dir: Optional output directory (default: docker_container/Workflow/). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - project_dir: Optional project root to sweep for extra .py helper files. + workflow_file: Path to the workflow Python file. + stub_files: List of stub file paths to include. + output_dir: Optional output directory (default: docker_container/Workflow/). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -487,7 +504,10 @@ def generate_workflow_docker( # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) # Copy gRPC generated stubs if they exist From 95240ca941167a11c6da6cc791ca4a2fb13c7ebe Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 19:17:55 -0700 Subject: [PATCH 03/31] ventis build: sweep project .py files into Docker build contexts generate_docker()/generate_workflow_docker() now recursively sweep every .py file under the project directory into the build context, preserving directory structure, so helper files that aren't declared as an agent entrypoint still make it into the image. Generated dirs (docker_container/, stubs/, grpc_stubs/) are excluded at the project root only, not at every depth. Stub files are placed at their agent's declared entrypoint path (mapped from global_controller.yaml) instead of a hardcoded guess, so a stub overwrites the exact real file it replaces. Guards against absolute and '..'-containing entrypoints, symlinked sources, and symlinked-destination escapes, with warnings on unsafe or unmapped stubs. Co-Authored-By: Claude Sonnet 5 --- ventis/cli.py | 45 +++++++++++++++--------------------- ventis/stub_generator.py | 49 ++++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index cb9ee0a..4c9badf 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -212,6 +212,23 @@ def cmd_build(args): if not yaml_files: logger.warning("No agent YAML files found in %s", agents_dir) + import yaml + + # Looks up a config entry's YAML and to map stubs to entrypoints. + yaml_by_name = {} + for yaml_path in yaml_files: + with open(yaml_path) as f: + name = yaml.safe_load(f).get("agent", {}).get("name") + if name: + yaml_by_name[name] = yaml_path + + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -220,23 +237,6 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) - # Map each stub's basename to its agent's declared entrypoint, so a stub - # overwrites the exact real file it replaces instead of guessing its path. - stub_entrypoints = {} - for agent_cfg in agents: - entrypoint = agent_cfg.get("entrypoint") - if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: - continue - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_cfg["name"]: - base_name = os.path.splitext(os.path.basename(yaml_path))[0] - stub_entrypoints[f"{base_name}.py"] = entrypoint - break - # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -310,16 +310,7 @@ def cmd_build(args): continue # Find matching YAML by agent name - matching_yaml = None - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_name: - matching_yaml = yaml_path - break - + matching_yaml = yaml_by_name.get(agent_name) if not matching_yaml: logger.warning( "No YAML definition found for agent '%s', skipping Docker", diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a56080d..9ea6c99 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -278,24 +278,43 @@ def _sweep_py_files(project_dir): and not (root == project_dir and d in _GENERATED_DIRS) ] for fname in files: - if fname.endswith(".py"): - abs_src = os.path.join(root, fname) + abs_src = os.path.join(root, fname) + if fname.endswith(".py") and not os.path.islink(abs_src): rel_dst = os.path.relpath(abs_src, project_dir) swept.append((abs_src, rel_dst)) return swept def _stub_destination(stub_file, stub_entrypoints): - """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" + """Where to copy a stub so it overwrites the real file it replaces, falling back to flat if that's unsafe.""" basename = os.path.basename(stub_file) entrypoint = stub_entrypoints.get(basename) if entrypoint: normalized = entrypoint.replace("\\", "/") if not normalized.startswith("/") and ".." not in normalized.split("/"): - return entrypoint + return normalized + print(f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat instead") + elif stub_entrypoints: + print(f" Warning: no entrypoint mapping for stub {basename}, placing flat instead") return basename +def _copy_files(output_dir, files_to_copy): + """Copy each (src, dst) pair into output_dir, refusing to write outside it (e.g. via a symlinked destination parent).""" + real_output_dir = os.path.realpath(output_dir) + for src, dst in files_to_copy: + if not os.path.isfile(src): + print(f" Warning: source file not found, skipping: {src}") + continue + dest_path = os.path.join(output_dir, dst) + real_dest = os.path.realpath(dest_path) + if os.path.commonpath([real_output_dir, real_dest]) != real_output_dir: + print(f" Warning: destination escapes build context, skipping: {dst}") + continue + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) + + def generate_docker( yaml_path, agent_file, @@ -386,13 +405,7 @@ def generate_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # Copy the YAML definition too shutil.copy2( @@ -477,9 +490,7 @@ def generate_workflow_docker( workflow_basename = os.path.basename(workflow_file) # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = [] - if project_dir: - files_to_copy += _sweep_py_files(project_dir) + files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), @@ -500,7 +511,7 @@ def generate_workflow_docker( for name in ("gpu_metrics.py", "session_logging.py") ], ] - + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( @@ -516,13 +527,7 @@ def generate_workflow_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading From 69d5405c24485ff51db960ad7843e496d91354ce Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 13:06:31 -0700 Subject: [PATCH 04/31] Fix missing os import in metrics_agent.py Co-Authored-By: Claude Sonnet 5 --- examples/portfolio/agents/metrics_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 374a869..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,6 +8,7 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. +import os import sys import json From 653bee84d8999a233bf71ce1e75e0ae2d1d44057 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 13:23:55 -0700 Subject: [PATCH 05/31] WIP: OTel exporter testing + portfolio merge-conflict fix (pre-pull checkpoint) Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 174 ++++++++++ OTel_Exporter/__init__.py | 0 OTel_Exporter/convert.py | 96 ++++++ OTel_Exporter/db.py | 185 ++++++++++ OTel_Exporter/otel_exporter.py | 98 ++++++ .../helloworld/workflow/example_workflow.py | 4 +- examples/portfolio/agents/advisor_agent.py | 16 +- examples/portfolio/agents/intent_agent.py | 35 -- examples/portfolio/agents/llm_agent.py | 50 --- examples/portfolio/agents/llm_agent.yaml | 14 - examples/portfolio/agents/metrics_agent.py | 1 + .../portfolio/config/global_controller.yaml | 32 +- examples/portfolio/config/policy.yaml | 1 - pyproject.toml | 6 +- requirements.txt | 4 + uv.lock | 320 ++++++++++++++++++ ventis/controller/global_controller.py | 57 +++- ventis/controller/utils/process_supervisor.py | 59 ++++ ventis/deploy.py | 2 +- ventis/stub_generator.py | 9 +- 20 files changed, 1020 insertions(+), 143 deletions(-) create mode 100644 OTel_Exporter/DESIGN.md create mode 100644 OTel_Exporter/__init__.py create mode 100644 OTel_Exporter/convert.py create mode 100644 OTel_Exporter/db.py create mode 100644 OTel_Exporter/otel_exporter.py delete mode 100644 examples/portfolio/agents/llm_agent.py delete mode 100644 examples/portfolio/agents/llm_agent.yaml create mode 100644 ventis/controller/utils/process_supervisor.py diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md new file mode 100644 index 0000000..73c4da2 --- /dev/null +++ b/OTel_Exporter/DESIGN.md @@ -0,0 +1,174 @@ +# OTLP Exporter for Ventis GlobalController — Design + +Status: **implemented (single-table design)**. `GlobalController` writes futures into a +`waiting` table (SQLite); a GC-supervised, GC-restarted OTel Exporter process reads +finished/unsent rows, converts each to an OTel span, and hands it to a real +`BatchSpanProcessor`/`OTLPSpanExporter`. Batching, serialization, and sending are all +OTel SDK code — the only custom pieces are the row→span conversion and durable +sent-tracking. This doc is a design/rationale reference; the actual files +(`otel_exporter.py`, `db.py`, `convert.py`, `ventis/controller/utils/process_supervisor.py`) +are the source of truth for current behavior. + +## Context +Ventis futures need to reach an external OTLP-compatible tracing backend. Design: a +separate OTLP Exporter process, spawned and supervised by GlobalController, that reads +unsent finished future rows from a local SQLite DB, converts them into OTel spans, and +hands them to the OTel SDK's own batching/export machinery, which ships them to an +external OTLP Receiver (out of scope here — assumed to be a separate, already-addressable +service). + +Decisions (final status): +- **Process model**: a true separate OS process, spawned and supervised by + GlobalController (not an in-process thread) — via `ProcessSupervisor` + (`ventis/controller/utils/process_supervisor.py`, built): `register`/`start_all` to + spawn, `check_and_respawn` (called from GC's existing poll tick, guarded on + `self.running` to avoid a shutdown race) to restart it if it ever dies unexpectedly, + `terminate_all` (called from GC's `stop()`) to shut it down cleanly. Rationale: fault + isolation from GC's core polling/health loop and independent restart, at low added + complexity since SQLite is already the entire hand-off boundary between the two. +- **Config**: implemented via a new `otel:` section in `global_controller.yaml` + (`protocol`/`endpoint`/`headers`), *not* by making `otel_exporter.py` itself + config-aware. `GlobalController` translates that section into the OTel SDK's own + standard env vars (`OTEL_EXPORTER_OTLP_PROTOCOL`/`_ENDPOINT`/`_HEADERS`) and passes + them to the exporter subprocess via `ProcessSupervisor.register(..., env=...)`. The + exporter still just constructs `OTLPSpanExporter()` with no explicit args (endpoint + and headers are resolved by the SDK itself from those env vars, same as always) and + reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly, to pick the gRPC vs HTTP exporter + class — the one piece of protocol selection the plain SDK classes don't do on their + own. Deliberately vendor-neutral: no backend name (Postgres, Langfuse, or otherwise) + appears anywhere in `otel_exporter.py`; the destination is 100% deploy-time config, + set once in `global_controller.yaml` and never touched by app code again. The + originally-planned `database.url` repurposing (below, kept for history) was decided + against — env-var configuration is the SDK's own idiomatic mechanism, so no + exporter-side config plumbing was added, only a GC-side YAML→env-var translation. + Does not (yet) support simultaneous multi-destination export — see "Known gaps". +- **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own + SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing + `_poll_controllers` *alongside* (not instead of) the existing + `send_runtime_information` write. Keeps this pipeline's schema/state fully decoupled + from the dashboard/cost table. +- **Two tables collapsed into one**: an earlier version of this design had a second + `queue` table (`waiting` → promote → `queue` → drain → send). Collapsed once it became + clear `BatchSpanProcessor` already provides its own in-memory queue — the only thing a + second table added was durability across the exporter's own process restarts, which a + `sent` column on `waiting` alone provides just as well, with less code. See `db.py`'s + module docstring. +- **Span construction**: settled — spans are built as `ReadableSpan` objects directly + (bypassing `Tracer`/`TracerProvider` entirely, no `IdGenerator` workaround needed for + either `trace_id` or `span_id`). Confirmed working via `ConsoleSpanExporter` during + development and via real (though unreachable) OTLP export attempts. + +## Implementation summary + +### 1. Config +`global_controller.yaml` gains an optional `otel:` section: +```yaml +otel: + protocol: grpc # or http + endpoint: otlp-pg-receiver.railway.internal:4317 + headers: {} # e.g. Authorization: "Basic " for a backend needing auth +``` +`GlobalController._otel_exporter_env()` translates this into +`OTEL_EXPORTER_OTLP_PROTOCOL`/`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` +and hands them to `ProcessSupervisor.register("otel_exporter", ..., env=...)`, which now +supports an `env` param (merged on top of the parent process's own environment, not a +replacement). Omitting `otel:` entirely falls back to whatever ambient env the exporter +subprocess would otherwise inherit, same as before this change. + +`otel_exporter.py` itself reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly (to pick +which `OTLPSpanExporter` class to import — gRPC or HTTP; the plain SDK classes don't +self-select this the way `opentelemetry-instrument`'s auto-config does). Endpoint and +headers are never read directly — `OTLPSpanExporter()` is still constructed with no +explicit args, letting the SDK resolve those from the same env vars itself, exactly as +before this change. `BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000)` +— the flush delay is explicitly overridden from the SDK default (5000ms) to 1000ms; +`max_export_batch_size` is left at the SDK default (512), which already approximates the +original "500 spans" batching ask without any override needed. + +### 2. `OTel_Exporter/otel_exporter.py` +A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM +stays responsive), calling `_send_pending()` each tick: +- `SELECT * FROM waiting WHERE finished_at IS NOT NULL AND (sent IS NULL OR sent = 0)`. +- Per row, each isolated in its own try/except (one malformed row is logged and skipped, + never blocks the rest of the batch): `convert.waiting_row_to_span(row)` → + `_processor.on_end(span)` → `db.mark_sent(future_id)` immediately — atomic per row, not + batched at the end, so a crash mid-poll can't leave an already-sent row unmarked (which + would cause a duplicate send on the next run). +- `_processor` is constructed once at startup; no `TracerProvider` is used at all, since + spans are hand-built and handed straight to the processor via `on_end()`. +- `_processor.shutdown()` on exit, flushing any pending batch. + +### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) +`future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is +the one that maps to `trace_id`. Both are `uuid4().hex` (32 hex chars / 16 bytes); OTel +`trace_id` is 128-bit (16 bytes, fits directly) and `span_id` is 64-bit (8 bytes, needs +truncation). No hashing — just hex-decode and truncate (deterministic, pure): +```python +trace_id = int(row["session_id"], 16) +span_id = int.from_bytes(bytes.fromhex(row["future_id"])[:8], "big") +parent_span_id = int.from_bytes(bytes.fromhex(row["parent_id"])[:8], "big") if row["parent_id"] else None +``` +Spans are assembled as plain `ReadableSpan(name=..., context=SpanContext(...), parent=SpanContext(...) or None, attributes=..., events=..., status=..., start_time=..., end_time=...)` +— no `Tracer`, no `IdGenerator`. Failed rows get a hand-built `exception` `Event` (using +the SDK's own `EXCEPTION_TYPE`/`EXCEPTION_MESSAGE` constants from `opentelemetry.sdk.trace`, +not hardcoded strings — `record_exception()` can't be used retrospectively since there's +no live exception object, only strings) plus `Status(StatusCode.ERROR, description=...)`. + +**Attribute naming**: `model`/`input_token_count`/`output_token_count` are set under the +real, current OTel GenAI semantic-convention keys — `gen_ai.request.model`/ +`gen_ai.usage.input_tokens`/`gen_ai.usage.output_tokens` — verified against the actual +spec (`open-telemetry/semantic-conventions`), not assumed. `cpu`/`gpu`/ +`execution_time_ms`/`queue_time_ms`/`token_count` keep plain names deliberately: none of +them have an OTel GenAI equivalent (cpu/gpu/queue-time are Ventis infra concepts, and +`token_count`, an input+output sum, isn't part of the spec at all — inventing a +`gen_ai.*`-shaped name for any of these would fabricate a standard rather than follow +one. `cached_tokens`/`cache_hit_ratio` exist on the `waiting` row but aren't exported to +attributes at all yet — a separate, pre-existing gap, not touched here. + +### 4. Process supervisor — `ventis/controller/utils/process_supervisor.py` (built) +`ProcessSupervisor`: `register(name, argv, env=None)` declares a process spec (`env`, +when given, is merged on top of — not a replacement for — the parent's own environment); +`start_all()` spawns everything registered; `check_and_respawn()` restarts anything that +exited, replaying the same argv/env (called from GC's `_poll_controllers`, guarded by +`if self.running:` so a SIGTERM mid-tick can't cause it to resurrect a process +`terminate_all()` just intentionally killed); `terminate_all()` terminates every managed +process (all `.terminate()` calls first, then `.wait()` on each, falling back to +`.kill()`), called from GC's `stop()`. Adding a future second daemon is one more +`register()` call — no new spawn/monitor/terminate code needed. + +### 5. Dependencies (all added) +`opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, +`opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` +config work, since `protocol: http` now needs that package importable). + +## Known gaps (not yet built) +- Spans carry no explicit `resource`/`instrumentation_scope` — would show as + `service.name=unknown_service` at a real backend. +- No simultaneous multi-destination export — `otel:` configures exactly one + destination; sending to two backends at once would mean registering a second, + separately-configured `otel_exporter` subprocess (same script, different env), not + something the exporter or its config format do today. +- `waiting` grows unboundedly: sent rows are never pruned, and futures that never finish + (`finished_at` never arrives) also stay forever, invisible and un-expiring. +- `error_name` is always `NULL` — Ventis's own Redis writer never records a distinct + exception-type field, only a message string. +- No committed test suite — all verification during development was ad hoc scripts, not + `pytest` files under `tests/`. +- Never verified against a live OTLP receiver — only against a refused connection + (confirmed the SDK's real retry/error-handling path is exercised correctly). +- No retry-limit/quarantine for a permanently malformed row — it logs an error every poll + forever rather than being given up on. + +## Verification approach used during development +- Row→span conversion: ad hoc scripts asserting deterministic id derivation, correct + parent/child linkage, correct `ERROR` status + `exception` event on failed rows, and + passing hand-built spans through `ConsoleSpanExporter().export([span])` to confirm the + SDK accepts them without error. +- Pipeline correctness: seeded `waiting` with mixes of finished/still-running/malformed/ + failed rows, ran the real `otel_exporter.py` subprocess, and inspected the resulting + `sent` flags and log output directly — including confirming a second run does not + re-send already-sent rows, and that a malformed row is skipped without blocking others. +- Process supervision: unit-tested `ProcessSupervisor` against a dummy process (spawn, + kill, confirm respawn with a new PID, confirm clean `terminate_all`) and + integration-tested it managing the real `otel_exporter.py` process. +- Scoped to the local provider throughout — no EC2 needed. diff --git a/OTel_Exporter/__init__.py b/OTel_Exporter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/OTel_Exporter/convert.py b/OTel_Exporter/convert.py new file mode 100644 index 0000000..8e062a4 --- /dev/null +++ b/OTel_Exporter/convert.py @@ -0,0 +1,96 @@ +"""Convert a `waiting` table row (see db.py) into an OTel ReadableSpan. + +Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects +directly instead of going through Tracer.start_span() -- there's no live tracer here, +futures already finished (sometimes in another process), so this is a historical-row +conversion, not live tracing. See OTel_Exporter/DESIGN.md for why this deviates from +the SDK's usual advice against constructing ReadableSpan by hand. +""" + +from opentelemetry.sdk.trace import EXCEPTION_MESSAGE, EXCEPTION_TYPE, Event, ReadableSpan +from opentelemetry.trace import SpanContext, SpanKind, TraceFlags +from opentelemetry.trace.status import Status, StatusCode + +_SAMPLED = TraceFlags(TraceFlags.SAMPLED) + + +def to_epoch_nanos(unix_seconds): + """Convert a unix-epoch-seconds float (as stored in waiting) to OTel's ns int.""" + if unix_seconds is None: + return None + return round(float(unix_seconds) * 1e9) + + +def waiting_row_to_span(row): + """Convert one waiting row (dict-like, column names as keys) into a ReadableSpan. + + Rows without finished_at are accepted but produce a span with end_time=None -- + filtering to finished rows is the caller's responsibility, not this function's. + """ + # sqlite3.Row supports row["col"] but not row.get("col") -- normalize once so the + # rest of this function can use .get() freely for optional fields. + row = dict(row) + + trace_id = int(row["session_id"], 16) + span_id = int.from_bytes(bytes.fromhex(row["future_id"])[:8], "big") + parent_id = row.get("parent_id") + parent_span_id = ( + int.from_bytes(bytes.fromhex(parent_id)[:8], "big") if parent_id else None + ) + + context = SpanContext( + trace_id=trace_id, span_id=span_id, is_remote=False, trace_flags=_SAMPLED + ) + parent = ( + SpanContext( + trace_id=trace_id, span_id=parent_span_id, is_remote=False, trace_flags=_SAMPLED + ) + if parent_span_id + else None + ) + + events = [] + status = Status(StatusCode.UNSET) + if row["failed"]: + events.append( + Event( + name="exception", + attributes={ + EXCEPTION_TYPE: row.get("error_name") or "RuntimeError", + EXCEPTION_MESSAGE: row.get("error_message") or "", + }, + timestamp=to_epoch_nanos(row.get("finished_at")), + ) + ) + status = Status(StatusCode.ERROR, description=row.get("error_message")) + + # model/input/output use real OTel GenAI semconv names; cpu/gpu/execution_time_ms/ + # queue_time_ms/token_count have no semconv equivalent (Ventis infra concepts, or -- + # for token_count -- a derived sum the spec doesn't define), so they keep plain names + # rather than being forced into a fake gen_ai.* one. See DESIGN.md. + attributes = { + k: v + for k, v in { + "gen_ai.request.model": row.get("model"), + "cpu": row.get("cpu"), + "gpu": row.get("gpu"), + "execution_time_ms": row.get("execution_time_ms"), + "queue_time_ms": row.get("queue_time_ms"), + "gen_ai.usage.input_tokens": row.get("input_token_count"), + "gen_ai.usage.output_tokens": row.get("output_token_count"), + "token_count": row.get("token_count"), + }.items() + if v is not None + } + + return ReadableSpan( + name=row.get("agent_id") or "unknown_agent", + context=context, + parent=parent, + attributes=attributes, + events=events, + status=status, + kind=SpanKind.INTERNAL, + start_time=to_epoch_nanos(row.get("started_at")), + end_time=to_epoch_nanos(row.get("finished_at")), + ) diff --git a/OTel_Exporter/db.py b/OTel_Exporter/db.py new file mode 100644 index 0000000..4bb301d --- /dev/null +++ b/OTel_Exporter/db.py @@ -0,0 +1,185 @@ +"""SQLite schema and writes for the OTel export pipeline's waiting table. + +`waiting` holds future rows as GlobalController observes them (including still-running +ones). There's no separate queue table -- OTel's own BatchSpanProcessor already queues +and batches spans in memory, so the only thing we need to track durably is which rows +have already been sent, which the `sent` column on this same table provides. (An earlier +version of this pipeline had a second `queue` table for that; collapsed away since it +wasn't doing anything BatchSpanProcessor doesn't already do -- see DESIGN.md.) +""" + +import os +import sqlite3 + +from ventis.controller.utils import pricing + +DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "otel_queue.db") + +# Demo-only multipliers for scaling displayed costs; not real recorded costs. Kept +# deliberately standalone/duplicated from telemetry_logging.py's identical constants +# (rather than importing them) so this module has no dependency on it -- keep these in +# sync by hand if the multipliers there ever change. +_TOKEN_COST_MULTIPLIER = 10000 +_SERVER_COST_MULTIPLIER = 100000 + +# Timestamps are stored as unix epoch seconds (matching the Redis future hash fields +# they're read from), not as SQLite datetime strings. Column set mirrors +# runtime_information 1:1 (see telemetry_logging.py) plus this pipeline's own additions +# (error_name/error_message/sent). +_TABLE_COLUMNS = """ + future_id TEXT PRIMARY KEY, + parent_id TEXT, + session_id TEXT NOT NULL, + project_id TEXT, + agent_id TEXT, + model TEXT, + cpu REAL, + gpu REAL, + started_at TIMESTAMP, + finished_at TIMESTAMP, + execution_time_ms INTEGER, + queue_time_ms INTEGER, + input_token_count INTEGER, + output_token_count INTEGER, + token_count INTEGER, + errors INTEGER, + failed BOOLEAN, + server_cost REAL, + token_cost REAL, + total_cost REAL, + cached_tokens INTEGER, + cache_hit_ratio REAL, + error_name TEXT, + error_message TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + sent BOOLEAN DEFAULT 0 +""" + + +def init_db(db_path=DB_PATH): + """Create the waiting table if it doesn't already exist.""" + conn = sqlite3.connect(db_path) + try: + conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") + conn.commit() + finally: + conn.close() + + +# `sent` is deliberately excluded here so re-upserting a waiting row (e.g. GC +# re-writing it from Redis) never resets it back to unsent. +_COLUMNS = [ + "future_id", "parent_id", "session_id", "project_id", "agent_id", "model", + "cpu", "gpu", "started_at", "finished_at", "execution_time_ms", "queue_time_ms", + "input_token_count", "output_token_count", "token_count", "errors", + "failed", "server_cost", "token_cost", "total_cost", + "cached_tokens", "cache_hit_ratio", "error_name", "error_message", +] + +_WAITING_UPSERT = """ + INSERT INTO waiting ({cols}) VALUES ({placeholders}) + ON CONFLICT(future_id) DO UPDATE SET {updates} +""".format( + cols=", ".join(_COLUMNS), + placeholders=", ".join(f":{c}" for c in _COLUMNS), + updates=", ".join(f"{c}=excluded.{c}" for c in _COLUMNS if c != "future_id"), +) + + +def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH): + """Upsert future rows (as returned by telemetry_logging.pull_runtime_information) + into the waiting table. Unlike runtime_information, rows without finished_at are + kept (not skipped) -- that's what "waiting" means here. `redis_client` is only used + to look up the executing agent's instance type for server-cost pricing, mirroring + send_runtime_information; pass None to skip cost lookups (server_cost stays 0).""" + if not rows: + return + conn = sqlite3.connect(db_path) + try: + for raw in rows: + fid = raw.get("future_id") + session_id = raw.get("request_id") + if not fid or not session_id: + continue + agent_id = raw.get("agent") + started_at = float(raw.get("created_at") or 0) or None + finished_at = float(raw["finished_at"]) if raw.get("finished_at") else None + execution_time_ms = ( + round((finished_at - started_at) * 1000) + if finished_at and started_at + else None + ) + input_token_count = int(float(raw.get("input_token_count") or 0)) + output_token_count = int(float(raw.get("output_token_count") or 0)) + token_count = int(float(raw.get("token_count") or 0)) + cached_tokens = int(float(raw.get("input_cache_tokens") or 0)) + + token_cost = ( + pricing.compute_token_cost( + raw.get("model"), input_token_count, output_token_count + ) + * _TOKEN_COST_MULTIPLIER + ) + # Server cost needs an elapsed duration -- only available once finished. + if finished_at and started_at: + server_cost = ( + pricing.compute_server_cost( + redis_client.get(f"agent:{agent_id}:instance_type") + if redis_client is not None and agent_id + else None, + finished_at - started_at, + ) + * _SERVER_COST_MULTIPLIER + ) + else: + server_cost = 0.0 + + conn.execute( + _WAITING_UPSERT, + { + "future_id": fid, + "parent_id": raw.get("parent") or None, + "session_id": session_id, + "project_id": project_id, + "agent_id": agent_id, + "model": raw.get("model"), + "cpu": float(raw.get("cpu_resource") or 0), + "gpu": float(raw.get("gpu_resource") or 0), + "started_at": started_at, + "finished_at": finished_at, + "execution_time_ms": execution_time_ms, + "queue_time_ms": ( + round(float(raw["queue_time"]) * 1000) + if raw.get("queue_time") + else None + ), + "input_token_count": input_token_count, + "output_token_count": output_token_count, + "token_count": token_count, + "errors": int(raw.get("errors") or 0), + "failed": bool(int(raw.get("failed") or 0)), + "server_cost": server_cost, + "token_cost": token_cost, + "total_cost": server_cost + token_cost, + "cached_tokens": cached_tokens, + "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, + "error_name": raw.get("error_name"), + "error_message": raw.get("error_message"), + }, + ) + conn.commit() + finally: + conn.close() + + +def mark_sent(future_id, db_path=DB_PATH): + """Mark one waiting row sent. Call this immediately after successfully handing its + span to the batch processor -- one row, one commit -- so a crash between two rows' + sends can't leave an already-sent row unmarked (which would cause a duplicate send + on the next run).""" + conn = sqlite3.connect(db_path) + try: + conn.execute("UPDATE waiting SET sent = 1 WHERE future_id = ?", (future_id,)) + conn.commit() + finally: + conn.close() diff --git a/OTel_Exporter/otel_exporter.py b/OTel_Exporter/otel_exporter.py new file mode 100644 index 0000000..b1c7f1a --- /dev/null +++ b/OTel_Exporter/otel_exporter.py @@ -0,0 +1,98 @@ +"""Entrypoint for the OTLP Exporter process. + +Each poll tick: read finished, not-yet-sent rows from `waiting`, convert each to a span, +hand it to a BatchSpanProcessor/OTLPSpanExporter, and mark it sent -- batching, OTLP +serialization, and sending are all the SDK's own code, not ours (see DESIGN.md). Each +row's send-and-mark-sent is atomic and happens immediately after its own successful +send, not batched at the end, so a crash mid-poll can't leave an already-sent row +unmarked (which would cause a duplicate send next run). `OTLPSpanExporter()` takes no +explicit endpoint/headers here -- it falls back to the SDK's own standard +`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` env vars, or localhost:4317, +per the SDK's own default behavior. GlobalController sets those env vars (plus +`OTEL_EXPORTER_OTLP_PROTOCOL`, which this module reads itself below to pick the gRPC vs +HTTP class) from `global_controller.yaml`'s `otel:` section when it spawns this process; +this file has no YAML/app-config awareness of its own, only standard OTel env vars -- +see DESIGN.md. +""" + +import logging +import os +import signal +import sqlite3 +import time + +# Protocol is the one thing the SDK's own exporter classes don't self-select from +# OTEL_EXPORTER_OTLP_PROTOCOL -- endpoint/headers/auth stay fully env-var-driven via +# each class's own defaults; see OTel_Exporter/DESIGN.md. +if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").startswith("http"): + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +else: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +import convert +import db + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +_running = True +_processor = None +POLL_INTERVAL_SECONDS = 5 + + +def _handle_shutdown(signum, frame): + global _running + _running = False + + +def _send_pending(): + """Convert and send each finished, not-yet-sent waiting row.""" + conn = sqlite3.connect(db.DB_PATH) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + "SELECT * FROM waiting WHERE finished_at IS NOT NULL " + "AND (sent IS NULL OR sent = 0)" + ).fetchall() + finally: + conn.close() + if not rows: + return + sent_count = 0 + for row in rows: + try: + span = convert.waiting_row_to_span(row) + _processor.on_end(span) + except Exception as e: + logger.error( + "Skipping waiting row %s -- failed to send: %s", row["future_id"], e + ) + continue + db.mark_sent(row["future_id"]) + sent_count += 1 + logger.info("Sent %d span(s) to the batch processor.", sent_count) + + +def main(): + global _processor + signal.signal(signal.SIGTERM, _handle_shutdown) + signal.signal(signal.SIGINT, _handle_shutdown) + db.init_db() + _processor = BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000) + logger.info("OTel exporter process started.") + last_poll = 0 + while _running: + if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + try: + _send_pending() + except Exception as e: + logger.warning("Poll cycle failed (non-fatal): %s", e) + last_poll = time.time() + time.sleep(1) + _processor.shutdown() + logger.info("OTel exporter process exiting.") + + +if __name__ == "__main__": + main() diff --git a/examples/helloworld/workflow/example_workflow.py b/examples/helloworld/workflow/example_workflow.py index 8bd4600..842fe80 100644 --- a/examples/helloworld/workflow/example_workflow.py +++ b/examples/helloworld/workflow/example_workflow.py @@ -15,11 +15,11 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from example_agent_stub import ExampleAgentStub +from example_agent import ExampleAgent def main(name: str = "World"): - agent = ExampleAgentStub() + agent = ExampleAgent() greeting = agent.hello(name=name) return {"greeting": greeting.value()} diff --git a/examples/portfolio/agents/advisor_agent.py b/examples/portfolio/agents/advisor_agent.py index 2db763f..5cc31ae 100644 --- a/examples/portfolio/agents/advisor_agent.py +++ b/examples/portfolio/agents/advisor_agent.py @@ -11,13 +11,10 @@ # If the LLM is unavailable (returns an empty string), it falls back to a # deterministic templated summary so the pipeline still returns. # -# Resource profile: cheap CPU; the LLM cost sits in LLMAgent, not here. +# Resource profile: cheap CPU, single call per request, on the critical path. -import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -from llm_agent import LLMAgent try: from ventis.llm.bedrock import call_bedrock except ImportError: @@ -27,16 +24,14 @@ class AdvisorAgent(object): def __init__(self): self.tools = [self.summarize] - self.llm = LLMAgent() + self.model_id = os.environ.get( + "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" + ) + self.region = os.environ.get("AWS_REGION", "us-east-1") def summarize(self, holdings: dict, metrics: dict, risk: dict) -> str: """Write a short plain-English briefing on the portfolio.""" prompt = self._build_prompt(holdings, metrics, risk) - text = self.llm.complete( - prompt=prompt, max_tokens=400, temperature=0.2 - ).value() - if not text: - print("AdvisorAgent: LLM returned no output; using templated summary.") try: response = call_bedrock( model_id=self.model_id, @@ -48,7 +43,6 @@ def summarize(self, holdings: dict, metrics: dict, risk: dict) -> str: except Exception as e: print(f"AdvisorAgent: Bedrock call failed ({e}); using templated summary.") return self._fallback_summary(metrics, risk) - return text def _build_prompt(self, holdings: dict, metrics: dict, risk: dict) -> str: lines = ["You are a portfolio analyst. Given the figures below, write a " diff --git a/examples/portfolio/agents/intent_agent.py b/examples/portfolio/agents/intent_agent.py index 972c8af..1eb15d7 100644 --- a/examples/portfolio/agents/intent_agent.py +++ b/examples/portfolio/agents/intent_agent.py @@ -7,17 +7,6 @@ # -> {"holdings": {"AAPL": 0.4, "MSFT": 0.35, "NVDA": 0.25}, # "lookback_days": 180} # -<<<<<<< HEAD -# The actual model call lives in the shared LLMAgent (remote, resolved via -# .value()) — this agent only builds the prompt and parses the result, so no -# Bedrock boilerplate lives here. If the LLM is unavailable or returns -# unparseable output, parse() raises: there is no fallback, the request fails -# loudly rather than guessing at the holdings. Weights are renormalized to 1.0. -# -# Resource profile: cheap CPU; the LLM cost sits in LLMAgent, not here. - -import sys -======= # Calls AWS Bedrock (Converse API) via ventis.llm.bedrock -- same pattern as # AdvisorAgent -- so token/cost telemetry gets recorded onto this execution's # future::metrics hash. Configure with env vars: @@ -31,20 +20,14 @@ # Resource profile: cheap CPU, single call per request, on the critical path # before the fan-out. ->>>>>>> remotes/origin/telemetry-signals import os import re import json -<<<<<<< HEAD -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -from llm_agent import LLMAgent -======= try: from ventis.llm.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock ->>>>>>> remotes/origin/telemetry-signals DEFAULT_LOOKBACK_DAYS = 365 @@ -52,15 +35,6 @@ class IntentAgent(object): def __init__(self): self.tools = [self.parse] -<<<<<<< HEAD - self.llm = LLMAgent() - - def parse(self, query: str) -> dict: - """Parse a natural-language portfolio request into holdings + lookback.""" - text = self.llm.complete( - prompt=self._build_prompt(query), max_tokens=300, temperature=0.0 - ).value() -======= self.model_id = os.environ.get( "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" ) @@ -75,7 +49,6 @@ def parse(self, query: str) -> dict: region=self.region, ) text = response["output"]["message"]["content"][0]["text"] ->>>>>>> remotes/origin/telemetry-signals if not text: raise ValueError("IntentAgent: LLM returned no output for the request.") @@ -148,15 +121,7 @@ def _sanitize(self, parsed: dict) -> dict: if __name__ == "__main__": -<<<<<<< HEAD - # Assumes the LLMAgent stub (Future-returning) is on the path, as it is - # inside the deployed pipeline. - agent = IntentAgent() - print(agent.parse( - "Analyze 40% Apple, 35% Microsoft and 25% Nvidia over the last 6 months" -======= agent = IntentAgent() print(agent.parse( query="Analyze 40% Apple, 35% Microsoft and 25% Nvidia over the last 6 months" ->>>>>>> remotes/origin/telemetry-signals )) diff --git a/examples/portfolio/agents/llm_agent.py b/examples/portfolio/agents/llm_agent.py deleted file mode 100644 index d42e4fb..0000000 --- a/examples/portfolio/agents/llm_agent.py +++ /dev/null @@ -1,50 +0,0 @@ -# LLM Agent -# -# Shared inference node. Owns all the AWS Bedrock (Converse API) plumbing so no -# other agent has to carry boto3 boilerplate — they just call complete(prompt) -# and get text back. Configure with env vars: -# BEDROCK_MODEL_ID (default: meta.llama3-8b-instruct-v1:0) -# AWS_REGION (default: us-east-1) -# -# On any failure (no boto3, no creds, model not enabled) it returns an empty -# string; callers decide how to degrade (templated summary, regex parse, etc.). -# -# Resource profile: LLM-bound. This is the only node that talks to Bedrock, so -# it's the natural place to scale inference capacity independently. - -import os - - -class LLMAgent(object): - def __init__(self): - self.tools = [self.complete] - self.model_id = os.environ.get( - "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" - ) - self.region = os.environ.get("AWS_REGION", "us-east-1") - - def complete( - self, prompt: str, max_tokens: int = 400, temperature: float = 0.2 - ) -> str: - """Run a single-turn completion on Bedrock; '' on any failure.""" - try: - import boto3 - - client = boto3.client("bedrock-runtime", region_name=self.region) - response = client.converse( - modelId=self.model_id, - messages=[{"role": "user", "content": [{"text": prompt}]}], - inferenceConfig={ - "maxTokens": max_tokens, - "temperature": temperature, - }, - ) - return response["output"]["message"]["content"][0]["text"] - except Exception as e: - print(f"LLMAgent: Bedrock call failed ({e}).") - return "" - - -if __name__ == "__main__": - agent = LLMAgent() - print(agent.complete("Say hello in one short sentence.", max_tokens=50)) \ No newline at end of file diff --git a/examples/portfolio/agents/llm_agent.yaml b/examples/portfolio/agents/llm_agent.yaml deleted file mode 100644 index ccc0095..0000000 --- a/examples/portfolio/agents/llm_agent.yaml +++ /dev/null @@ -1,14 +0,0 @@ -agent: - name: LLMAgent - functions: - - name: complete - description: Run a single-turn completion on Bedrock; '' on any failure. - arguments: - - name: prompt - type: str - - name: max_tokens - type: int - - name: temperature - type: float - returns: - type: str diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 374a869..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,6 +8,7 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. +import os import sys import json diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index 29e8de9..9a0a9a5 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -6,26 +6,6 @@ # reflect each stage's real cost so the scheduler has placement decisions to make. agents: -<<<<<<< HEAD - # Shared inference node. Owns all Bedrock plumbing; IntentAgent and - # AdvisorAgent delegate their model calls here. LLM-bound — scale replicas - # to match inference demand. - - name: LLMAgent - host: localhost - port: 8075 - redis_port: 6379 - replicas: 1 - resources: - cpu: 1 - memory: 512 - entrypoint: agents/llm_agent.py - - # Stage 0: parse the free-text request into structured holdings + lookback - # window (calls LLMAgent). Cheap CPU, one call per request, on the critical - # path before the fan-out. - - name: IntentAgent - host: localhost - port: 8076 # Stage 0: parse the free-text request into structured holdings + lookback # window (calls Bedrock directly via ventis.llm.bedrock). Cheap CPU, one # call per request, on the critical path before the fan-out. @@ -36,8 +16,6 @@ agents: cpu: 1 memory: 256 entrypoint: agents/intent_agent.py - - # Stage 0: price history fetch. Network/IO-bound, cheap CPU. Called by provider: EC2 instance_type: t3.micro @@ -107,3 +85,13 @@ redis: host: localhost port: 6379 db: 0 + +# EC2 defaults for `provider: EC2` replicas. +ec2: + region: us-east-1 + ami_id: ami-031ff6df47f26b546 + subnet_id: subnet-0638ac6d79d488124 + security_group_ids: + - sg-025daf3a98e06cef3 + ssh_user: ubuntu + ssh_private_key_path: ~/.ssh/ventis_ec2 diff --git a/examples/portfolio/config/policy.yaml b/examples/portfolio/config/policy.yaml index 573c91b..834c88e 100644 --- a/examples/portfolio/config/policy.yaml +++ b/examples/portfolio/config/policy.yaml @@ -14,7 +14,6 @@ rules: - match: {} access: - Workflow - - LLMAgent - IntentAgent - PriceAgent - MetricsAgent diff --git a/pyproject.toml b/pyproject.toml index 2efc1ab..8410b24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,10 @@ dependencies = [ "pyyaml", "flask", "psutil", + "opentelemetry-api>=1.44.0", + "opentelemetry-sdk>=1.44.0", + "opentelemetry-exporter-otlp-proto-grpc>=1.44.0", + "opentelemetry-exporter-otlp-proto-http>=1.44.0", ] [project.scripts] @@ -23,7 +27,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["ventis*"] +include = ["ventis*", "OTel_Exporter*"] [tool.setuptools.package-data] ventis = [ diff --git a/requirements.txt b/requirements.txt index b0dd97e..dd7a254 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,7 @@ ipython sqlalchemy psycopg[binary] psutil +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp-proto-grpc +opentelemetry-exporter-otlp-proto-http diff --git a/uv.lock b/uv.lock index 9b7b188..ba10707 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,11 @@ version = 1 revision = 3 requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version < '3.13'", +] [[package]] name = "async-timeout" @@ -48,6 +53,178 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3d/82/6f8fbbea47b773734ba0199643d2d851e5c2f75bc3699fe99db8af344d96/botocore-1.43.58-py3-none-any.whl", hash = "sha256:f516159f0732da8249206163ccea3bd1f82ad2a9d184fe6ed447e1abdba4330e", size = 15426503, upload-time = "2026-07-28T19:34:53.508Z" }, ] +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/aa/554e2614f38fc34c58ff1d0911ae8535ad2516440d5482d76fe59f1088b0/charset_normalizer-3.5.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa", size = 369072, upload-time = "2026-08-15T08:16:22.964Z" }, + { url = "https://files.pythonhosted.org/packages/03/6d/439231dfc3ccfa6f8c06477b7da2219cbd41a2de3d49084df8ec7b5100f2/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda", size = 251142, upload-time = "2026-08-15T08:16:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/55/53/7d819bd23a00ef45039146fa2cce1daa2f0771e758c5653ee1f6edac91ed/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45", size = 240714, upload-time = "2026-08-15T08:16:26.392Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2c/45847198c16f4b38090cc7423b2b6a9008e438704d8ab413211832498d31/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab", size = 279637, upload-time = "2026-08-15T08:16:27.961Z" }, + { url = "https://files.pythonhosted.org/packages/69/2b/d8be3523ddf9f0b0f3e56d1359034aa10653a4d11564c697f802b4775766/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a", size = 276543, upload-time = "2026-08-15T08:16:29.399Z" }, + { url = "https://files.pythonhosted.org/packages/32/cd/4f564b8f132de25db594efc706897069f016790cea63a5669c9df2675f64/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3", size = 261644, upload-time = "2026-08-15T08:16:30.722Z" }, + { url = "https://files.pythonhosted.org/packages/f5/e3/38b975422534a608f98c360e79c2f07c763d66dd4272300d45fb1fee54b0/charset_normalizer-3.5.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb", size = 259609, upload-time = "2026-08-15T08:16:32.248Z" }, + { url = "https://files.pythonhosted.org/packages/87/bd/fbc24d825c66f1c74f6ccdea3742c3d8354a4888e86d1315a197fee69061/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f", size = 252457, upload-time = "2026-08-15T08:16:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2d/918d0e98a0e679469ed05bb2d90c2088b4d315bb612969d8499f76fb5210/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea", size = 242240, upload-time = "2026-08-15T08:16:35.396Z" }, + { url = "https://files.pythonhosted.org/packages/20/c8/c36f6e0b2dfec351bd38cbc05362697e58bcd073d7dbd95154290c9714ce/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f", size = 280308, upload-time = "2026-08-15T08:16:36.825Z" }, + { url = "https://files.pythonhosted.org/packages/ca/7b/311b3e02e8c4092400c449c850a760d8c45d900983c83a70cc07208c551d/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182", size = 258679, upload-time = "2026-08-15T08:16:38.22Z" }, + { url = "https://files.pythonhosted.org/packages/b9/90/082cc45599c392f28c036a497f49e0634041a785fc3849c80ccf396d096f/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa", size = 277221, upload-time = "2026-08-15T08:16:39.62Z" }, + { url = "https://files.pythonhosted.org/packages/58/ad/b9aecf38d805cbcf84fa94f14c5d972a16561e20296a11dc799a5dcf3763/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818", size = 263799, upload-time = "2026-08-15T08:16:40.885Z" }, + { url = "https://files.pythonhosted.org/packages/b7/23/b38a20598d5a825f85d9d7636860e56ff0db1479f86497a6e485aa9326f7/charset_normalizer-3.5.1-cp310-cp310-win32.whl", hash = "sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20", size = 182037, upload-time = "2026-08-15T08:16:42.198Z" }, + { url = "https://files.pythonhosted.org/packages/d2/21/83fffb77864408b8bf0fe1ca603926401d6f8775a8e150b39aacc9958f8a/charset_normalizer-3.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d", size = 206030, upload-time = "2026-08-15T08:16:43.787Z" }, + { url = "https://files.pythonhosted.org/packages/86/2e/b93135b5034b1157fb29554b0d06d4844ce62282f0e0a14036f93d7ee2e7/charset_normalizer-3.5.1-cp310-cp310-win_arm64.whl", hash = "sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5", size = 185092, upload-time = "2026-08-15T08:16:45.177Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b6/034f6802e9c3f6418966cfabb7db8c9252cc2429c5098f41cc43af804149/charset_normalizer-3.5.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30", size = 363585, upload-time = "2026-08-15T08:16:46.646Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fa/6a7e2a7c4b5451912b8c417732df79574354443592a88d616de03da66ae5/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488", size = 251189, upload-time = "2026-08-15T08:16:48.287Z" }, + { url = "https://files.pythonhosted.org/packages/a4/c8/ab42b07cfd82e919f427fcfaa7c41abae8242833ad1aad66d42bae40b669/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22", size = 239724, upload-time = "2026-08-15T08:16:49.67Z" }, + { url = "https://files.pythonhosted.org/packages/e7/80/b9348b5d3041209f98b4cdad7655766369233f1d533f4f4f7558e9717bec/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731", size = 280078, upload-time = "2026-08-15T08:16:51.228Z" }, + { url = "https://files.pythonhosted.org/packages/82/38/083a24028304bc85bb9e376fed801178423dcbb67495f73b6ea0624e1894/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c", size = 276650, upload-time = "2026-08-15T08:16:52.625Z" }, + { url = "https://files.pythonhosted.org/packages/0d/35/731ac04aa0a097fc1c97f0994c375bdb230c6c96619db794208fe664e9ce/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8", size = 262325, upload-time = "2026-08-15T08:16:54.085Z" }, + { url = "https://files.pythonhosted.org/packages/f5/28/c2028e7021fb89c6e56868ed0e387b8e9aa811abdd2ab3208d6578d2c930/charset_normalizer-3.5.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486", size = 261140, upload-time = "2026-08-15T08:16:55.604Z" }, + { url = "https://files.pythonhosted.org/packages/28/f0/0c0ceec6d98b7daa62e361e418135d59685811d79ba11529aad5cdf15e84/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f", size = 252791, upload-time = "2026-08-15T08:16:57.103Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3e/48f4cd187b1c33189d86039e9cbe4f92c05454175504b44ff81806d4d1bf/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c", size = 240730, upload-time = "2026-08-15T08:16:58.418Z" }, + { url = "https://files.pythonhosted.org/packages/42/85/f9e22af69af67c54cce42be9455d9c81294f918b4ccc454db01f66efcac2/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18", size = 280791, upload-time = "2026-08-15T08:16:59.918Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4c/9044135f42127630b6fa742feb51256353f6ab87a78f2fdd1de3de955a7f/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5", size = 259598, upload-time = "2026-08-15T08:17:01.421Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ed/1dd7cfebb4e75812934c49ca3b79757d11948053f7937ab7070c151f3c55/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b", size = 278217, upload-time = "2026-08-15T08:17:02.782Z" }, + { url = "https://files.pythonhosted.org/packages/bf/eb/239c84503cc9e3ba6eb34686a24bc66e84f3924efdd7e38e751a19f6bc10/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6", size = 263417, upload-time = "2026-08-15T08:17:04.216Z" }, + { url = "https://files.pythonhosted.org/packages/37/ab/4e4510e1e288478e2c8333131d1c1382382ba8cd2165053c79e39d1da961/charset_normalizer-3.5.1-cp311-cp311-win32.whl", hash = "sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b", size = 181774, upload-time = "2026-08-15T08:17:05.58Z" }, + { url = "https://files.pythonhosted.org/packages/e3/57/32f0ccea59e8612057c61d6fd22ef2cb63cca93c9fe594094919696ac170/charset_normalizer-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9", size = 206653, upload-time = "2026-08-15T08:17:07.075Z" }, + { url = "https://files.pythonhosted.org/packages/17/d4/b65c433fc521e58b5f54293982a5e51c05cb5f2dd3f1c7a6acb65b75324e/charset_normalizer-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10", size = 185630, upload-time = "2026-08-15T08:17:08.502Z" }, + { url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" }, + { url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" }, + { url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" }, + { url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" }, + { url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" }, + { url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" }, + { url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" }, + { url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" }, + { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" }, + { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" }, + { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" }, + { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" }, + { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" }, + { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" }, + { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" }, + { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" }, + { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" }, + { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" }, + { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" }, + { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" }, + { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" }, + { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" }, + { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" }, + { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" }, + { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" }, + { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" }, + { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" }, + { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" }, + { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" }, + { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" }, + { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" }, + { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" }, + { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" }, + { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" }, + { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" }, + { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" }, + { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" }, + { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" }, + { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" }, + { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" }, + { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" }, + { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" }, + { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" }, + { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" }, + { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" }, + { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" }, + { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" }, + { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" }, + { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" }, + { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" }, + { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" }, + { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" }, + { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" }, + { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" }, + { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" }, + { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" }, + { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" }, + { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" }, + { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" }, + { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" }, + { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" }, +] + [[package]] name = "click" version = "8.4.2" @@ -86,6 +263,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/90/fb8f1c84537fbf210c1f53a53ae473a805f6599c5a40b93c1bbadd211f7a/googleapis_common_protos-1.75.2.tar.gz", hash = "sha256:8829a3d1e4508c5b7b9a6b9525f7fccff611f8531644579a76466c29295d4bb2", size = 154083, upload-time = "2026-08-25T19:19:13.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/5b/1c9e55363c3b1890a98cae813de5b4ea327845756cd8fb7ee690140c7eac/googleapis_common_protos-1.75.2-py3-none-any.whl", hash = "sha256:6b83302f554ea93a0f48409c7fc2050f954bcbcddb7e3a9c76d4a823cb22920e", size = 307002, upload-time = "2026-08-25T19:18:08.927Z" }, +] + [[package]] name = "greenlet" version = "3.5.4" @@ -280,6 +469,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/b1/d1f150b2ab3b4ae9932c05104fe1edbcb7fbf505587ea8db99e49341a05f/grpcio_tools-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:b8c9686b0c19f70b63d8d6cfeff5ad3480bdedecd60f14711fe43950f5397253", size = 1224199, upload-time = "2026-07-23T15:22:16.064Z" }, ] +[[package]] +name = "idna" +version = "3.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, +] + [[package]] name = "itsdangerous" version = "2.2.0" @@ -395,6 +593,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/47/80d9e9d468dc5de3af5096f5ccdb065fa4dd1470f74495cc53e59e397f47/opentelemetry_exporter_otlp_proto_grpc-1.44.0.tar.gz", hash = "sha256:40d1ae9e03fcc36de3cbac610cc99f35894938bff9cfd90fc4ec68bd85448463", size = 27225, upload-time = "2026-07-16T15:25:38.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/29/6ae42ba32b153ae0a44ae125f0caff2188bbe62d99c82d1768da30864e72/opentelemetry_exporter_otlp_proto_grpc-1.44.0-py3-none-any.whl", hash = "sha256:6a1a645ea182a2f59440c51fa8301d309f3324a8f9d65f8395584b064b67ee4e", size = 19624, upload-time = "2026-07-16T15:25:19.096Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -606,6 +903,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "s3transfer" version = "0.19.2" @@ -727,6 +1039,10 @@ dependencies = [ { name = "flask" }, { name = "grpcio" }, { name = "grpcio-tools" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "psutil" }, { name = "psycopg", extra = ["binary"] }, { name = "pyyaml" }, @@ -740,6 +1056,10 @@ requires-dist = [ { name = "flask" }, { name = "grpcio" }, { name = "grpcio-tools" }, + { name = "opentelemetry-api", specifier = ">=1.44.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.44.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.44.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.44.0" }, { name = "psutil" }, { name = "psycopg", extras = ["binary"] }, { name = "pyyaml" }, diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 4e416b4..496ae1a 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,6 +3,7 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit +import importlib.util import logging import signal import subprocess @@ -15,6 +16,7 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs +from ventis.controller.utils.process_supervisor import ProcessSupervisor from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -100,6 +102,31 @@ def __init__(self, config_path): self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() + # Spawn the OTLP exporter as a separate process (see OTel_Exporter/DESIGN.md), + # supervised so it gets restarted if it ever exits unexpectedly. + otel_exporter_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "OTel_Exporter", + ) + otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") + self.process_supervisor = ProcessSupervisor() + # `otel:` in global_controller.yaml maps straight to the OTel SDK's own + # standard env vars, not app-specific args -- the exporter subprocess itself + # stays a plain vendor-neutral OTel process; see OTel_Exporter/DESIGN.md. + otel_env = self._otel_exporter_env(self.config.get("otel", {})) + self.process_supervisor.register( + "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + ) + self.process_supervisor.start_all() + + # waiting table GC writes future data into (see OTel_Exporter/db.py); the + # exporter process itself calls init_db() to create the table. + otel_db_spec = importlib.util.spec_from_file_location( + "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") + ) + self._otel_db = importlib.util.module_from_spec(otel_db_spec) + otel_db_spec.loader.exec_module(self._otel_db) + # ------------------------------------------------------------------ # # Stale container cleanup # # ------------------------------------------------------------------ # @@ -143,6 +170,22 @@ def _load_config(config_path): with open(config_path, "r") as f: return yaml.safe_load(f) + @staticmethod + def _otel_exporter_env(otel_cfg): + """Translate global_controller.yaml's `otel:` section into standard OTLP env + vars for the exporter subprocess; returns None if `otel:` is absent/empty so + the subprocess falls back to the SDK's own defaults untouched.""" + env = {} + if otel_cfg.get("protocol"): + env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] + if otel_cfg.get("endpoint"): + env["OTEL_EXPORTER_OTLP_ENDPOINT"] = otel_cfg["endpoint"] + if otel_cfg.get("headers"): + env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( + f"{k}={v}" for k, v in otel_cfg["headers"].items() + ) + return env or None + @staticmethod def _get_replica_placements(ctrl): """Normalize replicas into a list of (host, port) placements.""" @@ -407,14 +450,25 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ + # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which + # terminates every managed process) via the signal handler before this line is + # reached -- without the guard, this could respawn a process just intentionally + # killed. See OTel_Exporter/DESIGN.md. + if self.running: + self.process_supervisor.check_and_respawn() + for instance in self.instance_manager.list_instances(): name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) send_runtime_information( - pull_runtime_information(node_redis), + future_rows, node_redis, self.config.get("database", {}).get("url"), ) @@ -654,6 +708,7 @@ def stop(self): self.running = False self._stop_docker_agents() self._stop_redis_containers() + self.process_supervisor.terminate_all() logger.info("Global controller shut down.") diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py new file mode 100644 index 0000000..8f5724c --- /dev/null +++ b/ventis/controller/utils/process_supervisor.py @@ -0,0 +1,59 @@ +"""Registry for OS processes GlobalController spawns and supervises. + +register() + start_all() spawn processes; check_and_respawn() (call from GC's existing +poll tick) restarts any that exit unexpectedly; terminate_all() (call from GC's shutdown +path) stops them all cleanly. Deliberately GC-agnostic -- callers are responsible for not +calling check_and_respawn() during their own shutdown (see OTel_Exporter/DESIGN.md's +shutdown-race note). +""" + +import logging +import os +import subprocess + +logger = logging.getLogger(__name__) + + +class ProcessSupervisor: + def __init__(self): + self._specs = {} # name -> (argv, env) tuple + self._procs = {} # name -> subprocess.Popen + + def register(self, name, argv, env=None): + """Declare a process to manage. Does not start it -- call start_all() once + everything is registered. `env`, if given, is merged on top of (not a + replacement for) this process's own environment, so the child still inherits + PATH etc.""" + self._specs[name] = (argv, env) + + def start_all(self): + for name, (argv, env) in self._specs.items(): + self._start(name, argv, env) + + def _start(self, name, argv, env=None): + merged_env = {**os.environ, **env} if env else None + self._procs[name] = subprocess.Popen(argv, env=merged_env) + + def check_and_respawn(self): + """Restart any registered process that has exited.""" + for name, proc in list(self._procs.items()): + if proc.poll() is not None: + logger.warning( + "Managed process %r exited (code %s), respawning", + name, + proc.returncode, + ) + argv, env = self._specs[name] + self._start(name, argv, env) + + def terminate_all(self, timeout=10): + """Terminate every managed process, falling back to kill() on timeout.""" + for proc in self._procs.values(): + proc.terminate() + for name, proc in self._procs.items(): + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + self._procs.clear() diff --git a/ventis/deploy.py b/ventis/deploy.py index b6ac721..148d3a4 100644 --- a/ventis/deploy.py +++ b/ventis/deploy.py @@ -9,7 +9,7 @@ import ventis def my_workflow(query: str): - finance = FinanceAgentStub() + finance = FinanceAgent() price = finance.get_stock_price(ticker=query) return {"price": price.value()} diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 8c956ef..d9480be 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -164,13 +164,12 @@ def _build_stub_class(agent_config): Build an AST node for the entire stub class. Generates a class like: - class FinanceAgentStub(object): + class FinanceAgent(object): def __init__(self): pass ...stub methods... """ - # class_name = agent_config["name"] + "Stub" - class_name = agent_config["name"] + class_name = agent_config["name"] functions = agent_config.get("functions", []) # __init__ method: simple pass, no gRPC setup needed. @@ -241,7 +240,7 @@ def generate_stub(yaml_path, output_path): with open(output_path, "w") as f: f.write(source) - class_name = agent_config["name"] + "Stub" + class_name = agent_config["name"] print(f"Generated stub class '{class_name}' -> {output_path}") return source @@ -521,7 +520,7 @@ def start_lc(): "-o", "--output", default=None, - help="Output path for the generated stub file (default: stubs/_stub.py)", + help="Output path for the generated stub file (default: stubs/.py)", ) parser.add_argument( "--agent-file", From 4af43ae3abb8da914050f9f83c61cbf2badebf17 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Tue, 25 Aug 2026 20:40:54 -0700 Subject: [PATCH 06/31] [Feature] Pass env / secrets into agent containers Users had no way to get API keys (OpenAI, Anthropic, embedding models) into an agent container. Add a top-level `env_file` key to global_controller.yaml pointing at a local .env file, which reaches every container as `docker run --env-file`. - resolve_env_file validates the path before anything launches, so a missing .env fails at deploy time instead of deep inside a container. Relative paths resolve against the project root, matching entrypoint. - env_file_args is a context manager owning the local-vs-remote decision and the cleanup, so both runtimes share one code path. Local containers read the original file; remote containers get a copy that is deleted as soon as `docker run` returns, whether or not it succeeded. - GlobalController._push_file streams the file over ssh under `umask 077` rather than scp, so the copy is never briefly world-readable and the secret never lands in a command line. _run_cmd's ssh options moved to a shared _ssh_args. --env-file is appended after the explicit -e VENTIS_* flags; Docker gives those precedence regardless of order, so a stray VENTIS_* line in someone's .env cannot break agent wiring. Closes #50 --- ventis/cli.py | 10 ++ .../cloud_provider_logic/EC2/_runtime.py | 12 ++- .../cloud_provider_logic/Local/_runtime.py | 12 ++- ventis/controller/global_controller.py | 94 +++++++++++++------ ventis/controller/utils/env_file.py | 92 ++++++++++++++++++ 5 files changed, 185 insertions(+), 35 deletions(-) create mode 100644 ventis/controller/utils/env_file.py diff --git a/ventis/cli.py b/ventis/cli.py index b43a6b3..6d85e1f 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -16,6 +16,8 @@ import subprocess import sys +from ventis.controller.utils.env_file import resolve_env_file + logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" @@ -397,6 +399,14 @@ def cmd_deploy(args): config = _load_config(config_path) project_dir = os.getcwd() + # Fail here rather than after a fleet of containers is already up without + # the API keys they need. + try: + resolve_env_file(config, base_dir=project_dir) + except ValueError as e: + logger.error("%s", e) + sys.exit(1) + _ensure_grpc_stubs_importable(project_dir) if any( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 4d5f766..9955fa2 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -23,6 +23,7 @@ import boto3 +from ventis.controller.utils.env_file import env_file_args from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.utils.redis_client import RedisClient @@ -285,8 +286,15 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) if project_id: cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) - cmd.append(image) - result = _controller._run_cmd(cmd, host, user=ssh_user) + + # User secrets from `env_file`. Explicit -e flags above still win over + # anything in the file. + with env_file_args( + _controller, host, ssh_user, container_name, is_local=False + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _controller._run_cmd(cmd, host, user=ssh_user) if result.returncode != 0: raise RuntimeError( f"SSH bootstrap failed on {host}: {(result.stderr or result.stdout or '').strip()}" diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index 963eef3..a387f7b 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -8,6 +8,8 @@ import logging +from ventis.controller.utils.env_file import env_file_args + logger = logging.getLogger(__name__) DEFAULT_HOST = "localhost" @@ -110,9 +112,15 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): cmd.extend(["--memory", f"{resources['memory']}m"]) if resources.get("gpu"): cmd.extend(["--gpus", str(resources["gpu"])]) - cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) + # User secrets from `env_file`. Explicit -e flags above still win, so a + # stray VENTIS_* line in someone's .env cannot break agent wiring. + with env_file_args( + _require_controller(), host, user, runtime_id, _is_local_host(host) + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _require_controller()._run_cmd(cmd, host, user) if result.returncode != 0: raise RuntimeError(f"Failed to launch {runtime_id}") diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 1e24f10..241daff 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -4,6 +4,7 @@ import atexit import logging +import shlex import signal import subprocess import threading @@ -16,6 +17,7 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs +from ventis.controller.utils.env_file import resolve_env_file from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -64,6 +66,9 @@ class GlobalController(object): def __init__(self, config_path): self.config_path = config_path self.config = self._load_config(config_path) + # Validate before launching anything: an agent that boots without its + # API keys fails deep inside a container, where it is expensive to debug. + self.env_file_path = resolve_env_file(self.config) redis_cfg = self.config.get("redis", {}) self.redis = RedisClient( @@ -174,6 +179,7 @@ def reload_config(self): """Reload the config file and rebuild the routing table.""" logger.info("Reloading config from %s", self.config_path) self.config = self._load_config(self.config_path) + self.env_file_path = resolve_env_file(self.config) self.controllers = self.config.get("agents", []) self.poll_interval = self.config.get("poll_interval", 5) assign_project_id(self.config.get("project_id", 0)) @@ -642,6 +648,28 @@ def _send(instance): # Runtime launching # # ------------------------------------------------------------------ # + def _ssh_args(self, host, user=None): + """Return the `ssh ... target` prefix used to reach a remote host.""" + ssh_key_path = os.path.expanduser( + self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/ventis_ec2") + ) + return [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "IdentitiesOnly=yes", + "-o", + "ConnectTimeout=10", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-i", + ssh_key_path, + f"{user}@{host}" if user else host, + ] + def _run_cmd(self, cmd, host, user=None): """ Run a command locally or on a remote host via SSH. @@ -656,41 +684,45 @@ def _run_cmd(self, cmd, host, user=None): """ is_local = _is_local_host(host) if is_local: - return subprocess.run( - cmd, capture_output=True, text=True, timeout=180 - ) - else: - ssh_key_path = os.path.expanduser( - self.config.get("ec2", {}).get( - "ssh_private_key_path", "~/.ssh/ventis_ec2" - ) - ) - ssh_target = f"{user}@{host}" if user else host - remote_cmd = " ".join(cmd) - if cmd and cmd[0] == "docker": - remote_cmd = f"sudo {remote_cmd}" - return subprocess.run( - [ - "ssh", - "-o", - "StrictHostKeyChecking=no", - "-o", - "IdentitiesOnly=yes", - "-o", - "ConnectTimeout=10", - "-o", - "ServerAliveInterval=10", - "-o", - "ServerAliveCountMax=3", - "-i", - ssh_key_path, - ssh_target, - remote_cmd, - ], + return subprocess.run(cmd, capture_output=True, text=True, timeout=180) + + remote_cmd = " ".join(cmd) + if cmd and cmd[0] == "docker": + remote_cmd = f"sudo {remote_cmd}" + return subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + capture_output=True, + text=True, + timeout=180, + ) + + def _push_file(self, local_path, remote_path, host, user=None): + """ + Copy a local file to a remote host over SSH. + + Streams the bytes through `cat` under `umask 077` rather than using + `scp`, so a secrets file is never briefly world-readable on the far + side. + + Returns: + subprocess.CompletedProcess + """ + remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + with open(local_path, "rb") as f: + result = subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + stdin=f, capture_output=True, text=True, timeout=180, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to copy {local_path} to {host}:{remote_path}: " + f"{(result.stderr or result.stdout or '').strip()}" ) + return result def launch_docker_agents(self): """Launch all configured runtimes through InstanceManager.""" diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py new file mode 100644 index 0000000..c83f770 --- /dev/null +++ b/ventis/controller/utils/env_file.py @@ -0,0 +1,92 @@ +""" +Pass user secrets (API keys and friends) into agent containers. + +The user points `env_file` in `config/global_controller.yaml` at a local +`.env` file. Containers on this machine read that file directly; containers +on a remote host get a short-lived 0600 copy. Either way the file reaches +Docker as `--env-file`. +""" + +import logging +import os +from contextlib import contextmanager + +logger = logging.getLogger(__name__) + +REMOTE_ENV_DIR = "/tmp" + + +def resolve_env_file(config, base_dir=None): + """ + Return the absolute path of the configured env file, or None when unset. + + Relative paths resolve against `base_dir` (default: the current working + directory), matching how `entrypoint` and `workflow_file` are resolved. + + Raises: + ValueError: the file is configured but unusable. Deploy should fail + here rather than start a fleet of agents with no API keys. + """ + raw = config.get("env_file") + if not raw: + return None + + path = os.path.expanduser(str(raw)) + if not os.path.isabs(path): + path = os.path.join(base_dir or os.getcwd(), path) + path = os.path.abspath(path) + + if not os.path.exists(path): + raise ValueError(f"env_file does not exist: {path} (from env_file: {raw})") + if not os.path.isfile(path): + raise ValueError(f"env_file is not a file: {path} (from env_file: {raw})") + if not os.access(path, os.R_OK): + raise ValueError(f"env_file is not readable: {path}") + return path + + +def remote_env_path(container_name): + """Where a remote host holds this container's copy of the env file.""" + return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + + +@contextmanager +def env_file_args(controller, host, user, container_name, is_local): + """ + Yield the `docker run` flags that hand the user's env file to a container. + + A container on this machine reads the original file. A container on a + remote host gets a 0600 copy, deleted as soon as the `with` body ends -- + success or failure, since by then the container holds the variables + itself. Keep that body tight around `docker run` so the copy is never + on the host longer than it has to be. + + Yields an empty list when no `env_file` is configured. + """ + env_file_path = getattr(controller, "env_file_path", None) + if not env_file_path: + yield [] + return + + if is_local: + yield ["--env-file", env_file_path] + return + + remote_path = remote_env_path(container_name) + controller._push_file(env_file_path, remote_path, host, user=user) + try: + yield ["--env-file", remote_path] + finally: + _remove_remote_copy(controller, remote_path, host, user) + + +def _remove_remote_copy(controller, remote_path, host, user): + """Delete a remote copy. Best effort -- never masks the caller's error.""" + try: + result = controller._run_cmd(["rm", "-f", remote_path], host, user=user) + if getattr(result, "returncode", 0) != 0: + logger.warning("Failed to delete env file copy %s on %s", remote_path, host) + except Exception as e: + logger.warning( + "Failed to delete env file copy %s on %s: %s", remote_path, host, e + ) From 087ee15b66e967937297580fc551c121c7301a20 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Wed, 26 Aug 2026 13:28:52 -0700 Subject: [PATCH 07/31] Harden the remote env file copy against a hostile /tmp Two holes in the remote staging path, both found reviewing the feature commit. `umask 077` only governs files the shell creates, and `>` follows symlinks -- so it did not actually guarantee a 0600 copy. The destination path is fully predictable (`/tmp/ventis-env-ventis-ec2--`), so a local user on the remote host could pre-create it world-readable, or point it at a file of their own, and collect the API keys. Remove whatever sits at the path before writing; `rm -f` unlinks a symlink rather than following it, so `cat >` then creates a fresh file under the umask. `_run_cmd` joins its argv with spaces and hands the result to a remote shell unquoted. `_push_file` quoted its path but the cleanup `rm` did not, so a container name containing a space split the `rm` into two arguments that matched nothing -- it exited 0 while the secrets file stayed on the host, and the returncode check logged nothing. Scrub the name down to [A-Za-z0-9_.-] in remote_env_path, which also closes the same gap in the `--env-file` argument and in any future use of that path. Still open, tracked separately: a push that dies mid-transfer can leave a copy behind, since the cleanup only covers the `docker run` that follows. On EC2 the instance is terminated on that path, which disposes of it. --- ventis/controller/global_controller.py | 9 ++++++++- ventis/controller/utils/env_file.py | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 241daff..0b30307 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -704,10 +704,17 @@ def _push_file(self, local_path, remote_path, host, user=None): `scp`, so a secrets file is never briefly world-readable on the far side. + Anything already sitting at the destination is removed first: `umask` + only governs files the shell creates, and `>` follows symlinks. Without + the `rm`, a local user on the remote host could pre-create the path + world-readable, or point it at a file of their own, and collect + whatever we write there. + Returns: subprocess.CompletedProcess """ - remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + quoted = shlex.quote(remote_path) + remote_cmd = f"umask 077; rm -f {quoted}; cat > {quoted}" with open(local_path, "rb") as f: result = subprocess.run( self._ssh_args(host, user) + [remote_cmd], diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py index c83f770..6cd77b6 100644 --- a/ventis/controller/utils/env_file.py +++ b/ventis/controller/utils/env_file.py @@ -9,11 +9,13 @@ import logging import os +import re from contextlib import contextmanager logger = logging.getLogger(__name__) REMOTE_ENV_DIR = "/tmp" +_UNSAFE_PATH_CHARS = re.compile(r"[^A-Za-z0-9_.-]") def resolve_env_file(config, base_dir=None): @@ -46,8 +48,17 @@ def resolve_env_file(config, base_dir=None): def remote_env_path(container_name): - """Where a remote host holds this container's copy of the env file.""" - return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + """ + Where a remote host holds this container's copy of the env file. + + The name is scrubbed down to a shell-safe alphabet. This path is + interpolated into remote commands that `_run_cmd` joins with spaces and + hands to a shell unquoted, so a container name carrying a space would + split the cleanup `rm` into two harmless arguments -- it would exit 0 + while the secrets stayed on the host, with nothing in the log to say so. + """ + safe_name = _UNSAFE_PATH_CHARS.sub("-", container_name) + return f"{REMOTE_ENV_DIR}/ventis-env-{safe_name}" @contextmanager From db0ba260c9cc1f88943d98836ce556fb7b458817 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:05:41 -0700 Subject: [PATCH 08/31] WIP: OTel multi-destination fan-out (Railway+Langfuse+Grafana) + cleanup-race fix (pre-pull checkpoint) Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 102 ++++-- OTel_Exporter/convert.py | 11 +- OTel_Exporter/db.py | 44 ++- OTel_Exporter/otel_exporter.py | 266 +++++++++++++-- .../portfolio/config/global_controller.yaml | 18 + pyproject.toml | 5 + tests/test_otel_exporter_fanout.py | 318 ++++++++++++++++++ tests/test_otel_exporter_fields.py | 99 ++++++ uv.lock | 128 +++++++ ventis/controller/global_controller.py | 173 +++++++++- 10 files changed, 1074 insertions(+), 90 deletions(-) create mode 100644 tests/test_otel_exporter_fanout.py create mode 100644 tests/test_otel_exporter_fields.py diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md index 73c4da2..86a2a95 100644 --- a/OTel_Exporter/DESIGN.md +++ b/OTel_Exporter/DESIGN.md @@ -1,6 +1,6 @@ # OTLP Exporter for Ventis GlobalController — Design -Status: **implemented (single-table design)**. `GlobalController` writes futures into a +Status: **implemented (single-table design; multi-destination fan-out in progress)**. `GlobalController` writes futures into a `waiting` table (SQLite); a GC-supervised, GC-restarted OTel Exporter process reads finished/unsent rows, converts each to an OTel span, and hands it to a real `BatchSpanProcessor`/`OTLPSpanExporter`. Batching, serialization, and sending are all @@ -41,7 +41,12 @@ Decisions (final status): originally-planned `database.url` repurposing (below, kept for history) was decided against — env-var configuration is the SDK's own idiomatic mechanism, so no exporter-side config plumbing was added, only a GC-side YAML→env-var translation. - Does not (yet) support simultaneous multi-destination export — see "Known gaps". + The initial multi-destination extension uses one `otel.destinations` list and one + independent exporter/`BatchSpanProcessor` pair per destination. gRPC and HTTP + destinations may be mixed in the same list. The legacy single-destination fields + remain supported through the original standard-environment-variable path. + Configuration is read at exporter startup; changing it requires a + GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing @@ -64,39 +69,47 @@ Decisions (final status): `global_controller.yaml` gains an optional `otel:` section: ```yaml otel: - protocol: grpc # or http - endpoint: otlp-pg-receiver.railway.internal:4317 - headers: {} # e.g. Authorization: "Basic " for a backend needing auth + destinations: + - name: railway + protocol: grpc # or http + endpoint: otlp-pg-receiver.railway.internal:4317 + headers: {} + - name: langfuse + protocol: http + endpoint: https://cloud.langfuse.com/api/public/otel/v1/traces + headers: {} # e.g. Authorization: "Basic " ``` -`GlobalController._otel_exporter_env()` translates this into -`OTEL_EXPORTER_OTLP_PROTOCOL`/`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` -and hands them to `ProcessSupervisor.register("otel_exporter", ..., env=...)`, which now -supports an `env` param (merged on top of the parent process's own environment, not a -replacement). Omitting `otel:` entirely falls back to whatever ambient env the exporter -subprocess would otherwise inherit, same as before this change. +`GlobalController._otel_exporter_env()` translates each destination into the exporter +process's destination configuration and hands it to `ProcessSupervisor.register( +"otel_exporter", ..., env=...)`, which supports an `env` param (merged on top of the +parent process's own environment, not a replacement). The legacy single-destination +`protocol`/`endpoint`/`headers` form remains valid and continues through the SDK's +standard OTLP environment variables. Omitting `otel:` entirely falls back to whatever +ambient env the exporter subprocess would otherwise inherit, same as before this +change. -`otel_exporter.py` itself reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly (to pick -which `OTLPSpanExporter` class to import — gRPC or HTTP; the plain SDK classes don't -self-select this the way `opentelemetry-instrument`'s auto-config does). Endpoint and -headers are never read directly — `OTLPSpanExporter()` is still constructed with no -explicit args, letting the SDK resolve those from the same env vars itself, exactly as -before this change. `BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000)` +`otel_exporter.py` parses the destination configuration at startup and constructs the +appropriate OTLP exporter for each entry (gRPC or HTTP), passing that destination's +endpoint and headers to the SDK. `BatchSpanProcessor(..., schedule_delay_millis=1000)` — the flush delay is explicitly overridden from the SDK default (5000ms) to 1000ms; `max_export_batch_size` is left at the SDK default (512), which already approximates the original "500 spans" batching ask without any override needed. ### 2. `OTel_Exporter/otel_exporter.py` A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM -stays responsive), calling `_send_pending()` each tick: +stays responsive), calling `_send_pending()` each tick. At startup it constructs one +independent OTLP exporter and `BatchSpanProcessor` for each configured destination; +each pair may use a different protocol, endpoint, and headers: - `SELECT * FROM waiting WHERE finished_at IS NOT NULL AND (sent IS NULL OR sent = 0)`. - Per row, each isolated in its own try/except (one malformed row is logged and skipped, never blocks the rest of the batch): `convert.waiting_row_to_span(row)` → - `_processor.on_end(span)` → `db.mark_sent(future_id)` immediately — atomic per row, not - batched at the end, so a crash mid-poll can't leave an already-sent row unmarked (which - would cause a duplicate send on the next run). -- `_processor` is constructed once at startup; no `TracerProvider` is used at all, since - spans are hand-built and handed straight to the processor via `on_end()`. -- `_processor.shutdown()` on exit, flushing any pending batch. + `on_end(span)` on every configured processor → `db.mark_sent(future_id)` immediately. + The row is marked after it has been queued to all processors. `sent` therefore means + **queued to every configured destination**, not remotely acknowledged; this is the + initial best-effort delivery contract and retains the existing single boolean schema. +- Each processor is constructed once at startup; no `TracerProvider` is used at all, + since spans are hand-built and handed straight to the processors via `on_end()`. +- Every processor is shut down on exit, flushing its pending batch independently. ### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) `future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is @@ -117,7 +130,11 @@ no live exception object, only strings) plus `Status(StatusCode.ERROR, descripti **Attribute naming**: `model`/`input_token_count`/`output_token_count` are set under the real, current OTel GenAI semantic-convention keys — `gen_ai.request.model`/ `gen_ai.usage.input_tokens`/`gen_ai.usage.output_tokens` — verified against the actual -spec (`open-telemetry/semantic-conventions`), not assumed. `cpu`/`gpu`/ +spec (`open-telemetry/semantic-conventions`), not assumed. Submitted `args` and the +completed `result` are stored in `waiting.input`/`waiting.output` as valid JSON text and +exported under Langfuse's documented `langfuse.observation.input`/ +`langfuse.observation.output` attributes. The span name is the stable logical +`service.method`, not the executing instance's UUID. `cpu`/`gpu`/ `execution_time_ms`/`queue_time_ms`/`token_count` keep plain names deliberately: none of them have an OTel GenAI equivalent (cpu/gpu/queue-time are Ventis infra concepts, and `token_count`, an input+output sum, isn't part of the spec at all — inventing a @@ -136,24 +153,45 @@ process (all `.terminate()` calls first, then `.wait()` on each, falling back to `.kill()`), called from GC's `stop()`. Adding a future second daemon is one more `register()` call — no new spawn/monitor/terminate code needed. -### 5. Dependencies (all added) +### 5. Poll/cleanup race fix (`ventis/controller/global_controller.py`) +GC's cleanup thread used to run on its own `cleanup_interval` timer (default 10s), +fully independent of the poll loop's `poll_interval` (default 5s) that writes futures +into `waiting`. On a fast-completing request, cleanup could delete a session's Redis +future keys before the next poll tick ever read them, so those futures never reached +`waiting` at all — silently dropped from every OTel destination, not just one. +Reproduced live: a fast request left only 1 of 6 agent calls in `waiting`. Fixed by +having the poll loop signal a `threading.Event` (`_cleanup_ready`) right after each +tick; the cleanup thread waits on that event instead of sleeping on its own timer, so +cleanup only ever runs immediately after a poll has already captured that tick's state. +Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is a +fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall +the poll loop's health checks and OTel writes. + +### 6. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). ## Known gaps (not yet built) +- `WriteResult()` passes an undefined `error_message` variable to its fan-out callback, + which can interrupt remote consumer propagation after the callback hash is persisted. +- Redis records failure text under `error`, but the waiting-table writer reads + `error_name`/`error_message`, so exported exception details are usually empty. +- Rows are marked `sent` immediately after `BatchSpanProcessor.on_end()` accepts them, + before the asynchronous OTLP export is confirmed; a later delivery failure can lose a + span while leaving `sent = 1`. - Spans carry no explicit `resource`/`instrumentation_scope` — would show as `service.name=unknown_service` at a real backend. -- No simultaneous multi-destination export — `otel:` configures exactly one - destination; sending to two backends at once would mean registering a second, - separately-configured `otel_exporter` subprocess (same script, different env), not - something the exporter or its config format do today. +- Destination-specific delivery acknowledgement/retry state is not tracked yet: + `sent` only records that the span was queued to all configured processors, so an + asynchronous export failure can still lose a span until a later delivery-state design + is added. - `waiting` grows unboundedly: sent rows are never pruned, and futures that never finish (`finished_at` never arrives) also stay forever, invisible and un-expiring. - `error_name` is always `NULL` — Ventis's own Redis writer never records a distinct exception-type field, only a message string. -- No committed test suite — all verification during development was ad hoc scripts, not - `pytest` files under `tests/`. +- Test coverage is still limited; the waiting-field migration/normalization/conversion + path is covered, but the exporter process and live OTLP delivery are not. - Never verified against a live OTLP receiver — only against a refused connection (confirmed the SDK's real retry/error-handling path is exercised correctly). - No retry-limit/quarantine for a permanently malformed row — it logs an error every poll diff --git a/OTel_Exporter/convert.py b/OTel_Exporter/convert.py index 8e062a4..b7fb493 100644 --- a/OTel_Exporter/convert.py +++ b/OTel_Exporter/convert.py @@ -64,10 +64,9 @@ def waiting_row_to_span(row): ) status = Status(StatusCode.ERROR, description=row.get("error_message")) - # model/input/output use real OTel GenAI semconv names; cpu/gpu/execution_time_ms/ - # queue_time_ms/token_count have no semconv equivalent (Ventis infra concepts, or -- - # for token_count -- a derived sum the spec doesn't define), so they keep plain names - # rather than being forced into a fake gen_ai.* one. See DESIGN.md. + # Model and token usage use OTel GenAI semantic-convention names. Observation + # input/output use Langfuse's documented JSON-string attributes. The remaining + # Ventis infrastructure values have no GenAI equivalent, so they keep plain names. attributes = { k: v for k, v in { @@ -79,12 +78,14 @@ def waiting_row_to_span(row): "gen_ai.usage.input_tokens": row.get("input_token_count"), "gen_ai.usage.output_tokens": row.get("output_token_count"), "token_count": row.get("token_count"), + "langfuse.observation.input": row.get("input"), + "langfuse.observation.output": row.get("output"), }.items() if v is not None } return ReadableSpan( - name=row.get("agent_id") or "unknown_agent", + name=row.get("name") or row.get("agent_id") or "unknown_agent", context=context, parent=parent, attributes=attributes, diff --git a/OTel_Exporter/db.py b/OTel_Exporter/db.py index 4bb301d..a8a675f 100644 --- a/OTel_Exporter/db.py +++ b/OTel_Exporter/db.py @@ -8,6 +8,7 @@ wasn't doing anything BatchSpanProcessor doesn't already do -- see DESIGN.md.) """ +import json import os import sqlite3 @@ -51,16 +52,31 @@ cache_hit_ratio REAL, error_name TEXT, error_message TEXT, + name TEXT, + input TEXT, + output TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, sent BOOLEAN DEFAULT 0 """ +_MIGRATION_COLUMNS = { + "name": "TEXT", + "input": "TEXT", + "output": "TEXT", +} + def init_db(db_path=DB_PATH): - """Create the waiting table if it doesn't already exist.""" + """Create the waiting table and add columns missing from older databases.""" conn = sqlite3.connect(db_path) try: conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") + existing_columns = { + row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() + } + for column, column_type in _MIGRATION_COLUMNS.items(): + if column not in existing_columns: + conn.execute(f"ALTER TABLE waiting ADD COLUMN {column} {column_type}") conn.commit() finally: conn.close() @@ -74,6 +90,7 @@ def init_db(db_path=DB_PATH): "input_token_count", "output_token_count", "token_count", "errors", "failed", "server_cost", "token_cost", "total_cost", "cached_tokens", "cache_hit_ratio", "error_name", "error_message", + "name", "input", "output", ] _WAITING_UPSERT = """ @@ -86,6 +103,17 @@ def init_db(db_path=DB_PATH): ) +def _normalize_json_text(value): + """Return JSON text, encoding legacy scalar strings that are not valid JSON.""" + if value is None or value == "": + return None + try: + json.loads(value) + except (json.JSONDecodeError, TypeError): + return json.dumps(value) + return value + + def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH): """Upsert future rows (as returned by telemetry_logging.pull_runtime_information) into the waiting table. Unlike runtime_information, rows without finished_at are @@ -113,6 +141,17 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH output_token_count = int(float(raw.get("output_token_count") or 0)) token_count = int(float(raw.get("token_count") or 0)) cached_tokens = int(float(raw.get("input_cache_tokens") or 0)) + service = raw.get("service") + method = raw.get("method") + name = raw.get("name") or ".".join( + part for part in (service, method) if part + ) + result = raw.get("result") + # Compatibility with pre-consolidation deployments, where completion + # metrics live in future:{id}:metrics but result lives in future:{id}. + # Unified hashes already include result and avoid this extra read. + if not result and finished_at and redis_client is not None: + result = redis_client.hget(f"future:{fid}", "result") token_cost = ( pricing.compute_token_cost( @@ -165,6 +204,9 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, "error_name": raw.get("error_name"), "error_message": raw.get("error_message"), + "name": name or agent_id or "unknown_agent", + "input": _normalize_json_text(raw.get("args")), + "output": _normalize_json_text(result), }, ) conn.commit() diff --git a/OTel_Exporter/otel_exporter.py b/OTel_Exporter/otel_exporter.py index b1c7f1a..cad2af8 100644 --- a/OTel_Exporter/otel_exporter.py +++ b/OTel_Exporter/otel_exporter.py @@ -1,33 +1,31 @@ -"""Entrypoint for the OTLP Exporter process. - -Each poll tick: read finished, not-yet-sent rows from `waiting`, convert each to a span, -hand it to a BatchSpanProcessor/OTLPSpanExporter, and mark it sent -- batching, OTLP -serialization, and sending are all the SDK's own code, not ours (see DESIGN.md). Each -row's send-and-mark-sent is atomic and happens immediately after its own successful -send, not batched at the end, so a crash mid-poll can't leave an already-sent row -unmarked (which would cause a duplicate send next run). `OTLPSpanExporter()` takes no -explicit endpoint/headers here -- it falls back to the SDK's own standard -`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` env vars, or localhost:4317, -per the SDK's own default behavior. GlobalController sets those env vars (plus -`OTEL_EXPORTER_OTLP_PROTOCOL`, which this module reads itself below to pick the gRPC vs -HTTP class) from `global_controller.yaml`'s `otel:` section when it spawns this process; -this file has no YAML/app-config awareness of its own, only standard OTel env vars -- -see DESIGN.md. +"""Entrypoint for the OTLP exporter process. + +Each poll tick reads finished, not-yet-sent rows from ``waiting``, converts each to a +span, hands it to every configured BatchSpanProcessor, and marks it sent only after +all processors accept it. Batching, OTLP serialization, and sending remain the SDK's +responsibility (see DESIGN.md). + +GlobalController may provide a JSON list in ``VENTIS_OTEL_DESTINATIONS``. That is a +Ventis-specific configuration because the standard OTEL exporter environment +variables describe only one destination. If it is absent, the original single +destination behavior is retained: the exporter class and its settings are selected +from the standard OTEL environment variables and SDK defaults. """ +import json import logging +import math import os import signal import sqlite3 import time -# Protocol is the one thing the SDK's own exporter classes don't self-select from -# OTEL_EXPORTER_OTLP_PROTOCOL -- endpoint/headers/auth stay fully env-var-driven via -# each class's own defaults; see OTel_Exporter/DESIGN.md. -if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").startswith("http"): - from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -else: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as GrpcOTLPSpanExporter, +) +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter as HttpOTLPSpanExporter, +) from opentelemetry.sdk.trace.export import BatchSpanProcessor import convert @@ -38,7 +36,165 @@ _running = True _processor = None +_processors = [] POLL_INTERVAL_SECONDS = 5 +DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" + + +def _normalize_protocol(protocol): + """Return the exporter family for a configured protocol name.""" + if not isinstance(protocol, str) or not protocol.strip(): + raise ValueError("destination protocol must be a non-empty string") + normalized = protocol.strip().lower().replace("_", "-") + if normalized in {"grpc", "otlp/grpc", "grpc/protobuf", "grpc-protobuf"}: + return "grpc" + if normalized in { + "http", + "http/protobuf", + "http-protobuf", + "http/proto", + "http+protobuf", + "protobuf", + }: + return "http" + raise ValueError( + f"unsupported destination protocol {protocol!r}; expected grpc or http/protobuf" + ) + + +def _validate_destination(destination, index): + if not isinstance(destination, dict): + raise ValueError(f"destination {index} must be an object") + + name = destination.get("name") + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"destination {index} name must be a non-empty string") + + protocol = _normalize_protocol(destination.get("protocol")) + endpoint = destination.get("endpoint") + if not isinstance(endpoint, str) or not endpoint.strip(): + raise ValueError(f"destination {name!r} endpoint must be a non-empty string") + + headers = destination.get("headers") + if headers is not None: + if not isinstance(headers, dict): + raise ValueError(f"destination {name!r} headers must be an object") + if any( + not isinstance(key, str) + or not key.strip() + or not isinstance(value, str) + for key, value in headers.items() + ): + raise ValueError( + f"destination {name!r} headers must map non-empty strings to strings" + ) + headers = dict(headers) + + insecure = destination.get("insecure") + if insecure is not None and not isinstance(insecure, bool): + raise ValueError(f"destination {name!r} insecure must be a boolean") + + timeout = destination.get("timeout") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ValueError(f"destination {name!r} timeout must be a positive number") + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError(f"destination {name!r} timeout must be a positive number") + + return { + "name": name.strip(), + "protocol": protocol, + "endpoint": endpoint.strip(), + "headers": headers, + "insecure": insecure, + "timeout": timeout, + } + + +def _configured_destinations(): + """Parse and validate the Ventis multi-destination environment variable. + + ``None`` means no Ventis-specific configuration was supplied, so callers can + preserve legacy OTEL environment-variable behavior. An empty or malformed value + is an explicit configuration error and fails startup rather than silently + exporting to the wrong destination. + """ + raw = os.environ.get(DESTINATIONS_ENV) + if raw is None: + return None + try: + destinations = json.loads(raw) + except (TypeError, json.JSONDecodeError) as exc: + raise ValueError(f"{DESTINATIONS_ENV} must contain a JSON list") from exc + if not isinstance(destinations, list) or not destinations: + raise ValueError(f"{DESTINATIONS_ENV} must contain a non-empty JSON list") + + validated = [] + names = set() + for index, destination in enumerate(destinations): + validated_destination = _validate_destination(destination, index) + name = validated_destination["name"] + if name in names: + raise ValueError(f"destination names must be unique; duplicate {name!r}") + names.add(name) + validated.append(validated_destination) + return validated + + +def _build_exporter(destination): + """Construct one explicitly configured exporter without logging credentials.""" + kwargs = { + "endpoint": destination["endpoint"], + } + if destination["headers"] is not None: + kwargs["headers"] = destination["headers"] + if destination["timeout"] is not None: + kwargs["timeout"] = destination["timeout"] + + if destination["protocol"] == "grpc": + if destination["insecure"] is not None: + kwargs["insecure"] = destination["insecure"] + return GrpcOTLPSpanExporter(**kwargs) + + if destination["insecure"] is not None: + logger.warning( + "Destination %s specifies insecure=%s, which is ignored for HTTP exporters.", + destination["name"], + destination["insecure"], + ) + return HttpOTLPSpanExporter(**kwargs) + + +def _build_processors(): + """Build destination processors, or one legacy processor when unconfigured.""" + destinations = _configured_destinations() + if destinations is None: + protocol = os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower() + exporter_class = ( + HttpOTLPSpanExporter if protocol.startswith("http") else GrpcOTLPSpanExporter + ) + return [("legacy", BatchSpanProcessor(exporter_class(), schedule_delay_millis=1000))] + + processors = [] + try: + for destination in destinations: + exporter = _build_exporter(destination) + processors.append( + ( + destination["name"], + BatchSpanProcessor(exporter, schedule_delay_millis=1000), + ) + ) + logger.info( + "Configured OTel destination %s (%s).", + destination["name"], + destination["protocol"], + ) + except Exception: + for _, processor in processors: + processor.shutdown() + raise + return processors def _handle_shutdown(signum, frame): @@ -48,6 +204,14 @@ def _handle_shutdown(signum, frame): def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" + processors = _processors + if not processors and _processor is not None: + # Compatibility for callers that configured the pre-fan-out singular + # ``_processor`` directly (the normal startup path always populates both). + processors = [("legacy", _processor)] + if not processors: + raise RuntimeError("OTel exporter has no configured processors") + conn = sqlite3.connect(db.DB_PATH) conn.row_factory = sqlite3.Row try: @@ -63,35 +227,63 @@ def _send_pending(): for row in rows: try: span = convert.waiting_row_to_span(row) - _processor.on_end(span) except Exception as e: logger.error( - "Skipping waiting row %s -- failed to send: %s", row["future_id"], e + "Skipping waiting row %s -- failed to convert: %s", row["future_id"], e ) continue + + failed_destinations = [] + for destination_name, processor in processors: + try: + processor.on_end(span) + except Exception as e: + # Still offer the span to the remaining processors. The row is only + # acknowledged when every destination accepted it, so a failed + # destination will be retried by the next poll. + failed_destinations.append(destination_name) + logger.error( + "Destination %s rejected waiting row %s: %s", + destination_name, + row["future_id"], + e, + ) + if failed_destinations: + continue db.mark_sent(row["future_id"]) sent_count += 1 - logger.info("Sent %d span(s) to the batch processor.", sent_count) + logger.info("Queued %d span(s) for all configured OTel destinations.", sent_count) def main(): - global _processor + global _processor, _processors signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _processor = BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000) - logger.info("OTel exporter process started.") - last_poll = 0 - while _running: - if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + _processors = _build_processors() + # Keep the old singular module variable available to integrations that imported + # it, while all sending uses the destination-aware collection above. + _processor = _processors[0][1] + logger.info("OTel exporter process started with %d destination(s).", len(_processors)) + try: + last_poll = 0 + while _running: + if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + try: + _send_pending() + except Exception as e: + logger.warning("Poll cycle failed (non-fatal): %s", e) + last_poll = time.time() + time.sleep(1) + finally: + for destination_name, processor in _processors: try: - _send_pending() + processor.shutdown() except Exception as e: - logger.warning("Poll cycle failed (non-fatal): %s", e) - last_poll = time.time() - time.sleep(1) - _processor.shutdown() - logger.info("OTel exporter process exiting.") + logger.error( + "Failed to shut down OTel destination %s: %s", destination_name, e + ) + logger.info("OTel exporter process exiting.") if __name__ == "__main__": diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index d61ad42..ab5af4d 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -78,6 +78,24 @@ agents: provider: EC2 instance_type: t3.micro + +otel: + destinations: + - name: railway + protocol: grpc + endpoint: yamanote.proxy.rlwy.net:19803 + insecure: true + headers: {} + - name: langfuse + protocol: http + endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces + headers: {} + - name: grafana + protocol: http + endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces + headers: + Authorization: Basic ${GRAFANA_OTLP_HEADERS} + # Polling interval in seconds poll_interval: 5 diff --git a/pyproject.toml b/pyproject.toml index 8410b24..9ae1234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,3 +58,8 @@ allowed-unresolved-imports = [ "*_stub", "*_agent_stub", ] + +[dependency-groups] +dev = [ + "pytest>=9.1.1", +] diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py new file mode 100644 index 0000000..ed1b423 --- /dev/null +++ b/tests/test_otel_exporter_fanout.py @@ -0,0 +1,318 @@ +"""Focused tests for the Ventis OTel exporter fan-out configuration.""" + +import json +import os +import sqlite3 +import sys +import tempfile +import types +import unittest +from unittest.mock import MagicMock, patch + + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +# ``otel_exporter.py`` is also executed as a script from its own directory and +# therefore imports ``convert`` and ``db`` as top-level modules. +sys.path.insert(0, os.path.join(ROOT, "OTel_Exporter")) + +import db # noqa: E402 +import otel_exporter # noqa: E402 + + +# The generated local-controller protobuf modules are build artifacts and are +# not present in a source checkout. The static config helper does not use them, +# so provide the tiny import-time surface needed to test it in isolation. +if "local_controler_pb2" not in sys.modules: + local_pb2 = types.ModuleType("local_controler_pb2") + local_pb2.JsonResponse = object + sys.modules["local_controler_pb2"] = local_pb2 +if "local_controler_pb2_grpc" not in sys.modules: + local_pb2_grpc = types.ModuleType("local_controler_pb2_grpc") + local_pb2_grpc.LocalControllerStub = object + sys.modules["local_controler_pb2_grpc"] = local_pb2_grpc + + +class OTelExporterFanoutTests(unittest.TestCase): + def setUp(self): + self.db_file = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + self.db_path = self.db_file.name + self.db_file.close() + db.init_db(self.db_path) + + def tearDown(self): + os.unlink(self.db_path) + + @staticmethod + def _destination_config(): + return [ + { + "name": "railway", + "protocol": "grpc", + "endpoint": "receiver.example:4317", + "headers": {"x-api-key": "railway-key"}, + "insecure": True, + "timeout": 3.5, + }, + { + "name": "langfuse", + "protocol": "http/protobuf", + "endpoint": "https://langfuse.example/api/public/otel", + "headers": {"authorization": "Basic secret"}, + "timeout": 7, + }, + ] + + def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): + grpc_exporter = object() + http_exporter = object() + grpc_processor = MagicMock(name="grpc_processor") + http_processor = MagicMock(name="http_processor") + destinations = self._destination_config() + + with patch.dict( + os.environ, + {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, + clear=True, + ), patch.object( + otel_exporter, + "GrpcOTLPSpanExporter", + return_value=grpc_exporter, + ) as grpc_constructor, patch.object( + otel_exporter, + "HttpOTLPSpanExporter", + return_value=http_exporter, + ) as http_constructor, patch.object( + otel_exporter, + "BatchSpanProcessor", + side_effect=[grpc_processor, http_processor], + ) as processor_constructor: + processors = otel_exporter._build_processors() + + self.assertEqual( + processors, [("railway", grpc_processor), ("langfuse", http_processor)] + ) + grpc_constructor.assert_called_once_with( + endpoint="receiver.example:4317", + headers={"x-api-key": "railway-key"}, + timeout=3.5, + insecure=True, + ) + http_constructor.assert_called_once_with( + endpoint="https://langfuse.example/api/public/otel", + headers={"authorization": "Basic secret"}, + timeout=7, + ) + self.assertEqual( + processor_constructor.call_args_list, + [ + unittest.mock.call(grpc_exporter, schedule_delay_millis=1000), + unittest.mock.call(http_exporter, schedule_delay_millis=1000), + ], + ) + + def test_build_processors_preserves_legacy_single_destination_fallback(self): + http_exporter = object() + processor = MagicMock(name="legacy_processor") + with patch.dict( + os.environ, + {"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, + clear=True, + ), patch.object( + otel_exporter, "HttpOTLPSpanExporter", return_value=http_exporter + ) as constructor, patch.object( + otel_exporter, "BatchSpanProcessor", return_value=processor + ): + result = otel_exporter._build_processors() + + self.assertEqual(result, [("legacy", processor)]) + constructor.assert_called_once_with() + + def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): + invalid_values = [ + "not-json", + json.dumps([]), + json.dumps( + [ + { + "name": "same", + "protocol": "grpc", + "endpoint": "one:4317", + }, + { + "name": "same", + "protocol": "http/protobuf", + "endpoint": "https://two", + }, + ] + ), + ] + for raw in invalid_values: + with self.subTest(raw=raw), patch.dict( + os.environ, {otel_exporter.DESTINATIONS_ENV: raw}, clear=True + ): + with self.assertRaises(ValueError): + otel_exporter._configured_destinations() + + def test_controller_expands_env_and_builds_langfuse_basic_auth(self): + from ventis.controller.global_controller import GlobalController + + with patch.dict( + os.environ, + { + "LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com", + "LANGFUSE_PUBLIC_KEY": "public", + "LANGFUSE_SECRET_KEY": "secret", + }, + clear=True, + ): + env = GlobalController._otel_exporter_env( + { + "destinations": [ + { + "name": "langfuse", + "protocol": "http/protobuf", + "endpoint": "${LANGFUSE_BASE_URL}/api/public/otel/v1/traces", + } + ] + } + ) + + destination = json.loads(env[otel_exporter.DESTINATIONS_ENV])[0] + self.assertEqual( + destination["endpoint"], + "https://us.cloud.langfuse.com/api/public/otel/v1/traces", + ) + self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") + + def test_controller_env_serializes_destinations_and_keeps_legacy_mapping(self): + # Importing the controller is intentionally local: this test remains + # runnable in the exporter-only environment used by the focused suite. + from ventis.controller.global_controller import GlobalController + + destinations = self._destination_config() + env = GlobalController._otel_exporter_env( + { + "protocol": "grpc", + "endpoint": "legacy.example:4317", + "headers": {"x-tenant": "demo"}, + "destinations": destinations, + } + ) + self.assertEqual(env["OTEL_EXPORTER_OTLP_PROTOCOL"], "grpc") + self.assertEqual(env["OTEL_EXPORTER_OTLP_ENDPOINT"], "legacy.example:4317") + self.assertEqual(env["OTEL_EXPORTER_OTLP_HEADERS"], "x-tenant=demo") + self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) + + def test_controller_rejects_invalid_destinations_before_starting_child(self): + from ventis.controller.global_controller import GlobalController + + invalid_destinations = [ + [], + [{"name": "railway", "protocol": "grpc"}], + [ + {"name": "same", "protocol": "grpc", "endpoint": "one:4317"}, + { + "name": "same", + "protocol": "http/protobuf", + "endpoint": "https://two", + }, + ], + [{"name": "bad", "protocol": "smtp", "endpoint": "example"}], + ] + for destinations in invalid_destinations: + with self.subTest(destinations=destinations), self.assertRaises(ValueError): + GlobalController._otel_exporter_env( + {"destinations": destinations} + ) + + def _insert_pending_row(self): + conn = sqlite3.connect(self.db_path) + try: + conn.execute( + """ + INSERT INTO waiting ( + future_id, session_id, started_at, finished_at, failed, + name, input, output, sent + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) + """, + ( + "00112233445566778899aabbccddeeff", + "ffeeddccbbaa99887766554433221100", + 1.0, + 2.0, + 0, + "PriceAgent.get_history", + '{"ticker":"NVDA"}', + '{"price":100}', + ), + ) + conn.commit() + finally: + conn.close() + + def test_send_pending_delivers_the_same_span_to_every_processor(self): + self._insert_pending_row() + first = MagicMock(name="first") + second = MagicMock(name="second") + with patch.object(otel_exporter.db, "DB_PATH", self.db_path), patch.object( + otel_exporter.db, "mark_sent" + ) as mark_sent: + otel_exporter._processors = [("railway", first), ("langfuse", second)] + otel_exporter._processor = None + otel_exporter._send_pending() + + first.on_end.assert_called_once() + second.on_end.assert_called_once() + self.assertIs(first.on_end.call_args.args[0], second.on_end.call_args.args[0]) + mark_sent.assert_called_once_with("00112233445566778899aabbccddeeff") + + def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_failure(self): + self._insert_pending_row() + failed = MagicMock(name="failed") + failed.on_end.side_effect = RuntimeError("destination unavailable") + remaining = MagicMock(name="remaining") + with patch.object(otel_exporter.db, "DB_PATH", self.db_path), patch.object( + otel_exporter.db, "mark_sent" + ) as mark_sent: + otel_exporter._processors = [("railway", failed), ("langfuse", remaining)] + otel_exporter._processor = None + otel_exporter._send_pending() + + failed.on_end.assert_called_once() + remaining.on_end.assert_called_once() + mark_sent.assert_not_called() + + conn = sqlite3.connect(self.db_path) + try: + self.assertEqual(conn.execute("SELECT sent FROM waiting").fetchone()[0], 0) + finally: + conn.close() + + def test_processor_construction_failure_shuts_down_already_built_processors(self): + first_processor = MagicMock(name="first_processor") + destinations = self._destination_config() + with patch.dict( + os.environ, + {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, + clear=True, + ), patch.object( + otel_exporter, + "GrpcOTLPSpanExporter", + return_value=object(), + ), patch.object( + otel_exporter, + "HttpOTLPSpanExporter", + side_effect=RuntimeError("bad HTTP exporter"), + ), patch.object( + otel_exporter, + "BatchSpanProcessor", + return_value=first_processor, + ): + with self.assertRaisesRegex(RuntimeError, "bad HTTP exporter"): + otel_exporter._build_processors() + + first_processor.shutdown.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py new file mode 100644 index 0000000..4a62c58 --- /dev/null +++ b/tests/test_otel_exporter_fields.py @@ -0,0 +1,99 @@ +import json +import os +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +from OTel_Exporter import convert, db + + +class OTelExporterFieldTests(unittest.TestCase): + def setUp(self): + handle = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + self.db_path = handle.name + handle.close() + + def tearDown(self): + os.unlink(self.db_path) + + def test_init_db_migrates_existing_waiting_table(self): + with sqlite3.connect(self.db_path) as conn: + conn.execute( + "CREATE TABLE waiting (future_id TEXT PRIMARY KEY, session_id TEXT NOT NULL)" + ) + + db.init_db(self.db_path) + + with sqlite3.connect(self.db_path) as conn: + columns = { + row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() + } + self.assertTrue({"name", "input", "output"}.issubset(columns)) + + def test_fields_are_normalized_and_added_to_span(self): + db.init_db(self.db_path) + raw = { + "future_id": "00112233445566778899aabbccddeeff", + "request_id": "ffeeddccbbaa99887766554433221100", + "service": "PriceAgent", + "method": "get_history", + "args": '{"ticker": "NVDA"}', + "result": "plain text result", + "created_at": "1.0", + "finished_at": "2.0", + "failed": "0", + } + + with patch.object(db.pricing, "compute_token_cost", return_value=0.0): + db.write_waiting_rows([raw], db_path=self.db_path) + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM waiting").fetchone() + + self.assertEqual(row["name"], "PriceAgent.get_history") + self.assertEqual(row["input"], raw["args"]) + self.assertEqual(json.loads(row["output"]), raw["result"]) + + span = convert.waiting_row_to_span(row) + self.assertEqual(span.name, "PriceAgent.get_history") + self.assertEqual(span.attributes["langfuse.observation.input"], raw["args"]) + self.assertEqual( + span.attributes["langfuse.observation.output"], row["output"] + ) + + def test_split_hash_result_is_loaded_for_finished_rows(self): + class SplitHashRedis: + def hget(self, key, field): + self.request = (key, field) + return '{"recommendation": "hold"}' + + def get(self, key): + return None + + db.init_db(self.db_path) + redis = SplitHashRedis() + raw = { + "future_id": "11112222333344445555666677778888", + "request_id": "88887777666655554444333322221111", + "service": "AdvisorAgent", + "method": "summarize", + "args": '{"risk": "moderate"}', + "result": "", + "created_at": "1.0", + "finished_at": "2.0", + "failed": "0", + } + + with patch.object(db.pricing, "compute_token_cost", return_value=0.0): + db.write_waiting_rows([raw], redis_client=redis, db_path=self.db_path) + + with sqlite3.connect(self.db_path) as conn: + output = conn.execute("SELECT output FROM waiting").fetchone()[0] + self.assertEqual(redis.request, (f"future:{raw['future_id']}", "result")) + self.assertEqual(json.loads(output), {"recommendation": "hold"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index ba10707..9b86805 100644 --- a/uv.lock +++ b/uv.lock @@ -246,6 +246,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + [[package]] name = "flask" version = "3.1.3" @@ -478,6 +490,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "itsdangerous" version = "2.2.0" @@ -692,6 +713,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -815,6 +854,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1003,6 +1069,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, ] +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" @@ -1050,6 +1170,11 @@ dependencies = [ { name = "sqlalchemy" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "boto3" }, @@ -1067,6 +1192,9 @@ requires-dist = [ { name = "sqlalchemy" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.1.1" }] + [[package]] name = "werkzeug" version = "3.1.8" diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index b84614b..9947680 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,15 +3,18 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit +import base64 import importlib.util +import json import logging +import math +import os +import re import signal import subprocess +import sys import threading import time -import json -import sys -import os from concurrent.futures import ThreadPoolExecutor import yaml @@ -102,7 +105,8 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread + # Start background cleanup thread, woken by each poll tick rather than its own timer -- see OTel_Exporter/DESIGN.md. + self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() @@ -114,22 +118,23 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # `otel:` in global_controller.yaml maps straight to the OTel SDK's own - # standard env vars, not app-specific args -- the exporter subprocess itself - # stays a plain vendor-neutral OTel process; see OTel_Exporter/DESIGN.md. + # Legacy `otel:` fields map straight to the OTel SDK's own standard env vars. + # A `destinations` list is additionally passed as one Ventis-specific JSON + # variable; the exporter subprocess remains a plain OTel process otherwise. otel_env = self._otel_exporter_env(self.config.get("otel", {})) self.process_supervisor.register( "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env ) - self.process_supervisor.start_all() - # waiting table GC writes future data into (see OTel_Exporter/db.py); the - # exporter process itself calls init_db() to create the table. + # Initialize/migrate the waiting table synchronously before either the GC or + # exporter process can access it. otel_db_spec = importlib.util.spec_from_file_location( "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") ) self._otel_db = importlib.util.module_from_spec(otel_db_spec) otel_db_spec.loader.exec_module(self._otel_db) + self._otel_db.init_db() + self.process_supervisor.start_all() # ------------------------------------------------------------------ # # Stale container cleanup # @@ -177,15 +182,50 @@ def _cleanup_stale_containers(self): @staticmethod def _load_config(config_path): - """Load the YAML config file.""" + """Load the YAML config file after importing root .env values.""" + project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) + GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: return yaml.safe_load(f) + @staticmethod + def _load_dotenv(path): + """Load simple KEY=VALUE entries without overriding existing environment values.""" + if not os.path.isfile(path): + return + with open(path, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + if key and key not in os.environ: + os.environ[key] = value + + @staticmethod + def _expand_otel_value(value): + if isinstance(value, str): + return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), value) + if isinstance(value, dict): + return {key: GlobalController._expand_otel_value(item) for key, item in value.items()} + if isinstance(value, list): + return [GlobalController._expand_otel_value(item) for item in value] + return value + @staticmethod def _otel_exporter_env(otel_cfg): """Translate global_controller.yaml's `otel:` section into standard OTLP env - vars for the exporter subprocess; returns None if `otel:` is absent/empty so - the subprocess falls back to the SDK's own defaults untouched.""" + vars for the exporter subprocess. When present, ``destinations`` is passed as + JSON for the exporter to construct a fan-out. Returns None if `otel:` is + absent/empty so the subprocess falls back to the SDK's own defaults untouched. + + The legacy protocol/endpoint/headers mappings intentionally remain unchanged + for existing configurations. + """ env = {} if otel_cfg.get("protocol"): env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] @@ -195,8 +235,109 @@ def _otel_exporter_env(otel_cfg): env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( f"{k}={v}" for k, v in otel_cfg["headers"].items() ) + + if "destinations" in otel_cfg: + destinations = GlobalController._expand_otel_value(otel_cfg["destinations"]) + for destination in destinations: + if destination.get("name") == "langfuse": + public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") + secret_key = os.environ.get("LANGFUSE_SECRET_KEY") + if public_key and secret_key: + auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode() + destination.setdefault("headers", {})["Authorization"] = f"Basic {auth}" + GlobalController._validate_otel_destinations(destinations) + try: + env["VENTIS_OTEL_DESTINATIONS"] = json.dumps(destinations) + except (TypeError, ValueError) as exc: + # Do not include the offending value: destination configs commonly + # contain credentials in headers. + raise ValueError( + "otel.destinations must contain JSON-serializable values" + ) from exc return env or None + @staticmethod + def _validate_otel_destinations(destinations): + """Validate the shape of the optional exporter fan-out configuration. + + Keep this validation deliberately structural: destination-specific options + are interpreted by the exporter. Error messages identify only the location + and type, never destination contents or header values. + """ + if not isinstance(destinations, list): + raise ValueError("otel.destinations must be a list") + if not destinations: + raise ValueError("otel.destinations must not be empty") + + names = set() + for index, destination in enumerate(destinations): + if not isinstance(destination, dict): + raise ValueError( + f"otel.destinations[{index}] must be a mapping" + ) + + for field in ("name", "protocol", "endpoint"): + value = destination.get(field) + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"otel.destinations[{index}].{field} must be a non-empty string" + ) + + name = destination["name"].strip() + if name in names: + raise ValueError(f"otel.destinations contains duplicate name {name!r}") + names.add(name) + + protocol = destination["protocol"].strip().lower().replace("_", "-") + if protocol not in { + "grpc", + "otlp/grpc", + "grpc/protobuf", + "grpc-protobuf", + "http", + "http/protobuf", + "http-protobuf", + "http/proto", + "http+protobuf", + "protobuf", + }: + raise ValueError( + f"otel.destinations[{index}].protocol must be grpc or http/protobuf" + ) + + if "headers" in destination: + headers = destination["headers"] + if not isinstance(headers, dict): + raise ValueError( + f"otel.destinations[{index}].headers must be a mapping" + ) + if any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in headers.items() + ): + raise ValueError( + f"otel.destinations[{index}].headers keys and values must be strings" + ) + + if "insecure" in destination and not isinstance( + destination["insecure"], bool + ): + raise ValueError( + f"otel.destinations[{index}].insecure must be a boolean" + ) + + if "timeout" in destination: + timeout = destination["timeout"] + if ( + isinstance(timeout, bool) + or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) + or timeout <= 0 + ): + raise ValueError( + f"otel.destinations[{index}].timeout must be a positive number" + ) + @staticmethod def _get_replica_placements(ctrl): """Normalize replicas into a list of (host, port) placements.""" @@ -479,6 +620,7 @@ def run(self): self._poll_controllers() except Exception as e: logger.warning("Polling loop encountered an error: %s", e) + self._cleanup_ready.set() time.sleep(self.poll_interval) except KeyboardInterrupt: self.stop() @@ -638,9 +780,10 @@ def _get_lc_stub(self, endpoint): return self._lc_stubs[endpoint] def _cleanup_loop(self): - """Background thread: periodically trigger cleanup of completed requests.""" + """Background thread: trigger cleanup right after each poll tick, or every cleanup_interval as a fallback.""" while True: - time.sleep(self.cleanup_interval) + self._cleanup_ready.wait(timeout=self.cleanup_interval) + self._cleanup_ready.clear() try: self._trigger_cleanup() except Exception as e: From 098548c60ee297032ba7cb7c3074726955917dd3 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:24:08 -0700 Subject: [PATCH 09/31] Dedupe 'import os' from PR #51 merge (both sides added it independently) Co-Authored-By: Claude Sonnet 5 --- examples/portfolio/agents/metrics_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index d5f722e..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -10,7 +10,6 @@ # Resource profile: cheap CPU, high fan-out — one compute() call per holding. import os import sys -import os import json import math From 0b9546cbe324a99b34589e5e5b497cdad09951e4 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:49:27 -0700 Subject: [PATCH 10/31] Fix PR #51 regression: disable entrypoint-based stub relocation This project's agents/workflow import each other's stubs by flat module name, not by the exporting agent's own entrypoint path. Applying _stub_destination's entrypoint-mirroring broke both the Workflow (ModuleNotFoundError: intent_agent) and agent-to-agent calls (MetricsAgent -> price_agent) on live redeploy. Keeps PR #51's actual fix (project_dir sweep for unstubbed helper files) intact. Co-Authored-By: Claude Sonnet 5 --- ventis/cli.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 28c4fd7..920df15 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -229,8 +229,8 @@ def cmd_build(args): logger.warning("No agent YAML files found in %s", agents_dir) import yaml - - # Looks up a config entry's YAML and to map stubs to entrypoints. + + # Looks up a config entry's YAML by agent name. yaml_by_name = {} for yaml_path in yaml_files: with open(yaml_path) as f: @@ -238,13 +238,6 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path - entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} - stub_entrypoints = { - f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] - for n, p in yaml_by_name.items() - if entrypoints_by_name.get(n) - } - stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -309,7 +302,9 @@ def cmd_build(args): api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + # A workflow script imports stubs by flat module name (e.g. `from + # intent_agent import ...`), not by the agent's own entrypoint path. + stub_entrypoints=None, ) else: @@ -345,7 +340,9 @@ def cmd_build(args): stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + # Same reasoning as the workflow call above: this project's agents + # import each other's stubs by flat module name, not entrypoint path. + stub_entrypoints=None, ) bake_targets.append( From 7b3b167b27e9e8889edf099de517dc82f5662a13 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 21:40:17 -0700 Subject: [PATCH 11/31] Parallelize per-instance polling in _poll_controllers Metrics/telemetry latency scaled with instance count x per-instance round-trip time since every instance was polled sequentially, one blocking the next, with the following tick only starting after the whole pass finished. Extracted the per-instance body into _poll_one_instance (whole body wrapped in one top-level try/except, since ThreadPoolExecutor.map() re-raises on first exception when results are consumed) and run all instances concurrently via the same ThreadPoolExecutor pattern _trigger_cleanup already used. Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 16 +- ventis/controller/global_controller.py | 217 +++++++++++++------------ 2 files changed, 130 insertions(+), 103 deletions(-) diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md index 86a2a95..418ca2a 100644 --- a/OTel_Exporter/DESIGN.md +++ b/OTel_Exporter/DESIGN.md @@ -167,7 +167,21 @@ Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall the poll loop's health checks and OTel writes. -### 6. Dependencies (all added) +### 6. Parallelized per-instance polling (`ventis/controller/global_controller.py`) +`_poll_controllers` used to loop over every instance sequentially -- Redis reads, an +OTel sqlite write, and up to two Postgres writes per instance, one instance fully +blocking the next, with the following poll tick only starting after the whole pass +finished. Total metrics/telemetry latency scaled with instance count x round-trip +time, not the configured `poll_interval`. Fixed by extracting the per-instance body +into `_poll_one_instance` (its whole body wrapped in one top-level try/except, since +`ThreadPoolExecutor.map()` re-raises on first exception when results are consumed) +and running all instances concurrently via the same `ThreadPoolExecutor` pattern +`_trigger_cleanup` already used. Known, pre-existing, previously acknowledged in +commit `a6694d9`'s own message but never actually fixed (a same-named follow-up +branch was found to contain no real threading changes) -- see company-memory for +the investigation. + +### 7. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index bdec683..da29783 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -643,118 +643,131 @@ def _poll_controllers(self): if self.running: self.process_supervisor.check_and_respawn() - for instance in self.instance_manager.list_instances(): + # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer + # gates every other instance's poll -- see OTel_Exporter/DESIGN.md. + instances = self.instance_manager.list_instances() + if instances: + with ThreadPoolExecutor(max_workers=len(instances)) as executor: + list(executor.map(self._poll_one_instance, instances)) + + def _poll_one_instance(self, instance): + """Poll and persist one instance's runtime/metrics/health data; never raises.""" + try: name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, + except Exception as e: + logger.warning("Failed to poll instance %s: %s", instance, e) + return + + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" + + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now - - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, - ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) - else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status + else: + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From b5d6e4dcbc6aee7265e1dc01080f70da39eb90f2 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 28 Aug 2026 13:58:05 -0700 Subject: [PATCH 12/31] rough draft --- .../portfolio/config/global_controller.yaml | 25 +++++---- .../portfolio/workflow/portfolio_workflow.py | 1 - pyproject.toml | 2 +- requirements.txt | 2 - tests/test_otel_exporter_fanout.py | 2 +- tests/test_otel_exporter_fields.py | 2 +- .../OTLP_Exporter}/DESIGN.md | 6 +- ventis/OTLP_Exporter/SCHEMA.md | 56 +++++++++++++++++++ .../OTLP_Exporter}/__init__.py | 0 .../OTLP_Exporter}/convert.py | 24 ++++++-- {OTel_Exporter => ventis/OTLP_Exporter}/db.py | 4 +- .../OTLP_Exporter}/otel_exporter.py | 0 ventis/OTLP_Exporter/otel_queue.db | 0 ventis/cli.py | 21 +++++-- .../cloud_provider_logic/EC2/_runtime.py | 5 +- ventis/controller/global_controller.py | 27 +++++---- ventis/controller/utils/process_supervisor.py | 2 +- ventis/stub_generator.py | 40 +++++++++++-- 18 files changed, 167 insertions(+), 52 deletions(-) rename {OTel_Exporter => ventis/OTLP_Exporter}/DESIGN.md (98%) create mode 100644 ventis/OTLP_Exporter/SCHEMA.md rename {OTel_Exporter => ventis/OTLP_Exporter}/__init__.py (100%) rename {OTel_Exporter => ventis/OTLP_Exporter}/convert.py (72%) rename {OTel_Exporter => ventis/OTLP_Exporter}/db.py (98%) rename {OTel_Exporter => ventis/OTLP_Exporter}/otel_exporter.py (100%) create mode 100644 ventis/OTLP_Exporter/otel_queue.db diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index ab5af4d..edee85f 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -83,18 +83,18 @@ otel: destinations: - name: railway protocol: grpc - endpoint: yamanote.proxy.rlwy.net:19803 + endpoint: ${RAILWAY_OTLP_ENDPOINT} insecure: true headers: {} - - name: langfuse - protocol: http - endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces - headers: {} - name: grafana protocol: http endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} +# - name: langfuse +# protocol: http +# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces +# headers: {} # Polling interval in seconds poll_interval: 5 @@ -107,10 +107,13 @@ redis: # EC2 defaults for `provider: EC2` replicas. ec2: - region: us-east-1 - ami_id: ami-031ff6df47f26b546 - subnet_id: subnet-0638ac6d79d488124 + region: ${EC2_REGION} + ami_id: ${EC2_AMI_ID} + subnet_id: ${EC2_SUBNET_ID} security_group_ids: - - sg-025daf3a98e06cef3 - ssh_user: ubuntu - ssh_private_key_path: ~/.ssh/ventis_ec2 + - ${EC2_SECURITY_GROUP_ID} + ssh_user: ${EC2_SSH_USER} + ssh_private_key_path: ${EC2_SSH_PRIVATE_KEY_PATH} + +database: + url: ${DATABASE_URL} diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 299a3f5..b8b684a 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -46,7 +46,6 @@ def main( advisor = AdvisorAgent() # Stage 0: parse the free-text request into structured holdings + window. - intent = intent_agent.parse(query=query).value() # parse() returns a dict, but a Future's .value() only ever gives back the # raw string ventis stored in Redis -- same deserialization requirement as # every other dict-returning agent call below. diff --git a/pyproject.toml b/pyproject.toml index 9ae1234..40cb43a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["ventis*", "OTel_Exporter*"] +include = ["ventis*"] [tool.setuptools.package-data] ventis = [ diff --git a/requirements.txt b/requirements.txt index dd7a254..f06e0b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,6 @@ grpcio-tools redis pyyaml flask -ipdb -ipython sqlalchemy psycopg[binary] psutil diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index ed1b423..0691d61 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -13,7 +13,7 @@ ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # ``otel_exporter.py`` is also executed as a script from its own directory and # therefore imports ``convert`` and ``db`` as top-level modules. -sys.path.insert(0, os.path.join(ROOT, "OTel_Exporter")) +sys.path.insert(0, os.path.join(ROOT, "ventis", "OTLP_Exporter")) import db # noqa: E402 import otel_exporter # noqa: E402 diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index 4a62c58..ff8cad6 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -5,7 +5,7 @@ import unittest from unittest.mock import patch -from OTel_Exporter import convert, db +from ventis.OTLP_Exporter import convert, db class OTelExporterFieldTests(unittest.TestCase): diff --git a/OTel_Exporter/DESIGN.md b/ventis/OTLP_Exporter/DESIGN.md similarity index 98% rename from OTel_Exporter/DESIGN.md rename to ventis/OTLP_Exporter/DESIGN.md index 418ca2a..d433fe0 100644 --- a/OTel_Exporter/DESIGN.md +++ b/ventis/OTLP_Exporter/DESIGN.md @@ -48,7 +48,7 @@ Decisions (final status): Configuration is read at exporter startup; changing it requires a GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own - SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing + SQLite file (`ventis/OTLP_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing `send_runtime_information` write. Keeps this pipeline's schema/state fully decoupled from the dashboard/cost table. @@ -95,7 +95,7 @@ endpoint and headers to the SDK. `BatchSpanProcessor(..., schedule_delay_millis= `max_export_batch_size` is left at the SDK default (512), which already approximates the original "500 spans" batching ask without any override needed. -### 2. `OTel_Exporter/otel_exporter.py` +### 2. `ventis/OTLP_Exporter/otel_exporter.py` A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM stays responsive), calling `_send_pending()` each tick. At startup it constructs one independent OTLP exporter and `BatchSpanProcessor` for each configured destination; @@ -111,7 +111,7 @@ each pair may use a different protocol, endpoint, and headers: since spans are hand-built and handed straight to the processors via `on_end()`. - Every processor is shut down on exit, flushing its pending batch independently. -### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) +### 3. Future row → OTel span conversion (`ventis/OTLP_Exporter/convert.py`) `future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is the one that maps to `trace_id`. Both are `uuid4().hex` (32 hex chars / 16 bytes); OTel `trace_id` is 128-bit (16 bytes, fits directly) and `span_id` is 64-bit (8 bytes, needs diff --git a/ventis/OTLP_Exporter/SCHEMA.md b/ventis/OTLP_Exporter/SCHEMA.md new file mode 100644 index 0000000..38f8664 --- /dev/null +++ b/ventis/OTLP_Exporter/SCHEMA.md @@ -0,0 +1,56 @@ +# OTel span storage schema + +Each emitted OTel span is stored as one `otel_spans` record. This chart also defines a +one-to-one `otel_span_attributes` projection, linked by the same `span_id`, for the +known exporter attributes. The current receiver retains the raw `attributes` JSONB map; +the child table is the relational schema described in `otel_spans_schema.txt`. + +```text +┌───────────────────────────┐ ┌────────────────────────────────┐ +│ otel_spans │ │ otel_span_attributes │ +├───────────────────────────┤ ├────────────────────────────────┤ +│ PK span_id │────1:1───│ PK/FK span_id │ +│ trace_id │ │ model and agent ID │ +│ parent_span_id │ │ CPU and GPU │ +│ name │ │ timing and token usage │ +│ kind │ │ input and output │ +│ start/end time (ns) │ │ project and error count │ +│ status code/message │ │ server/token/total cost │ +│ attributes (JSONB) │ │ cache tokens/hit ratio │ +│ events (JSONB) │ └────────────────────────────────┘ +└───────────────────────────┘ +``` + +## `otel_spans` + +| Column | Meaning | +| --- | --- | +| `span_id` | Unique identifier for this span. | +| `trace_id` | Identifier shared by all spans in the same trace. | +| `parent_span_id` | Parent span; empty for a root span. | +| `name` | Operation name, such as an agent method. | +| `kind` | OTel role of the work; current exporter spans are `SPAN_KIND_INTERNAL`. | +| `start_time_unix_nano` / `end_time_unix_nano` | Raw Unix timestamps in nanoseconds. | +| `status_code` / `status_message` | OTel outcome: normally `STATUS_CODE_UNSET`, or `STATUS_CODE_ERROR` with an error message. | +| `attributes` | Complete raw OTel attribute map (JSONB). | +| `events` | OTel events, including any `exception` event (JSONB). | + +## `otel_span_attributes` + +This one-to-one projection mirrors every attribute currently emitted by +`convert.waiting_row_to_span()`. Fields are nullable because OTel omits an attribute +whose source value is `None`. + +| Group | Columns | +| --- | --- | +| Model and agent | `gen_ai.request.model`, `gen_ai.agent.id` | +| Resources and timing | `cpu`, `gpu`, `execution_time_ms`, `queue_time_ms` | +| Token usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `token_count`, `gen_ai.usage.cache_read.input_tokens` | +| Input and output | `langfuse.observation.input`, `langfuse.observation.output` | +| Project and errors | `project_id`, `error_count` | +| Costs | `server_cost`, `token_cost`, `gen_ai.usage.cost` | +| Cache | `cache_hit_ratio` | + +The DBML source is [`../otel_spans_schema.txt`](../otel_spans_schema.txt). + +Successful spans use `STATUS_CODE_UNSET` rather than `STATUS_CODE_OK` because OpenTelemetry reserves `OK` for application- or operator-validated success, while instrumentation normally sets a status only when it records an error. diff --git a/OTel_Exporter/__init__.py b/ventis/OTLP_Exporter/__init__.py similarity index 100% rename from OTel_Exporter/__init__.py rename to ventis/OTLP_Exporter/__init__.py diff --git a/OTel_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py similarity index 72% rename from OTel_Exporter/convert.py rename to ventis/OTLP_Exporter/convert.py index b7fb493..66da972 100644 --- a/OTel_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -3,7 +3,7 @@ Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects directly instead of going through Tracer.start_span() -- there's no live tracer here, futures already finished (sometimes in another process), so this is a historical-row -conversion, not live tracing. See OTel_Exporter/DESIGN.md for why this deviates from +conversion, not live tracing. See ventis/OTLP_Exporter/DESIGN.md for why this deviates from the SDK's usual advice against constructing ReadableSpan by hand. """ @@ -64,9 +64,17 @@ def waiting_row_to_span(row): ) status = Status(StatusCode.ERROR, description=row.get("error_message")) - # Model and token usage use OTel GenAI semantic-convention names. Observation - # input/output use Langfuse's documented JSON-string attributes. The remaining - # Ventis infrastructure values have no GenAI equivalent, so they keep plain names. + # Model, token, agent, and cache-read usage use OTel GenAI semantic-convention + # names (see https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/). + # total_cost uses gen_ai.usage.cost, which isn't yet an official semconv attribute + # but is the name Langfuse's OTel ingestion actually reads (its JSON cost_details + # attribute is currently broken -- see langfuse/langfuse#11030). Observation + # input/output use Langfuse's documented JSON-string attributes. `errors` is named + # error_count, not "errors"/"error", to avoid colliding with OTel's reserved + # error.* namespace (error.type etc.), which describes a single error, not a + # count. The remaining Ventis-specific values (project_id, server/token cost + # breakdown, cache_hit_ratio) have no GenAI or Langfuse equivalent, so they keep + # plain names. attributes = { k: v for k, v in { @@ -80,6 +88,14 @@ def waiting_row_to_span(row): "token_count": row.get("token_count"), "langfuse.observation.input": row.get("input"), "langfuse.observation.output": row.get("output"), + "project_id": row.get("project_id"), + "gen_ai.agent.id": row.get("agent_id"), + "error_count": row.get("errors"), + "server_cost": row.get("server_cost"), + "token_cost": row.get("token_cost"), + "gen_ai.usage.cost": row.get("total_cost"), + "gen_ai.usage.cache_read.input_tokens": row.get("cached_tokens"), + "cache_hit_ratio": row.get("cache_hit_ratio"), }.items() if v is not None } diff --git a/OTel_Exporter/db.py b/ventis/OTLP_Exporter/db.py similarity index 98% rename from OTel_Exporter/db.py rename to ventis/OTLP_Exporter/db.py index a8a675f..e6a1834 100644 --- a/OTel_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -130,7 +130,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH if not fid or not session_id: continue agent_id = raw.get("agent") - started_at = float(raw.get("created_at") or 0) or None + started_at = float(raw.get("created_at") or 0) finished_at = float(raw["finished_at"]) if raw.get("finished_at") else None execution_time_ms = ( round((finished_at - started_at) * 1000) @@ -160,7 +160,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH * _TOKEN_COST_MULTIPLIER ) # Server cost needs an elapsed duration -- only available once finished. - if finished_at and started_at: + if finished_at is not None: server_cost = ( pricing.compute_server_cost( redis_client.get(f"agent:{agent_id}:instance_type") diff --git a/OTel_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py similarity index 100% rename from OTel_Exporter/otel_exporter.py rename to ventis/OTLP_Exporter/otel_exporter.py diff --git a/ventis/OTLP_Exporter/otel_queue.db b/ventis/OTLP_Exporter/otel_queue.db new file mode 100644 index 0000000..e69de29 diff --git a/ventis/cli.py b/ventis/cli.py index 920df15..31a32e0 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -238,6 +238,15 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path + # Maps each generated stub's basename to its agent's entrypoint path, so a + # stub can also be placed at its nested, entrypoint-mirrored location. + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -302,9 +311,9 @@ def cmd_build(args): api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # A workflow script imports stubs by flat module name (e.g. `from - # intent_agent import ...`), not by the agent's own entrypoint path. - stub_entrypoints=None, + # Stubs are placed both flat and at their entrypoint-mirrored path, + # so both flat and nested import styles resolve to the stub. + stub_entrypoints=stub_entrypoints, ) else: @@ -340,9 +349,9 @@ def cmd_build(args): stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # Same reasoning as the workflow call above: this project's agents - # import each other's stubs by flat module name, not entrypoint path. - stub_entrypoints=None, + # Same reasoning as the workflow call above: stubs are placed both + # flat and at their entrypoint-mirrored path. + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 9955fa2..7e4e9ab 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -239,11 +239,12 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, logger.info("Transferring image %s to %s", image, host) result = subprocess.run( "set -o pipefail; " - f"docker save {shlex.quote(image)} | ssh -o StrictHostKeyChecking=no " + f"docker save {shlex.quote(image)} | zstd -T0 | ssh -o StrictHostKeyChecking=no " f"-o IdentitiesOnly=yes -o ConnectTimeout=10 " f"-o ServerAliveInterval=10 -o ServerAliveCountMax=3 " f"-i {shlex.quote(key)} " - f"{shlex.quote(f'{ssh_user}@{host}')} 'sudo docker load'", + f"{shlex.quote(f'{ssh_user}@{host}')} " + "'set -o pipefail; zstd -d | sudo docker load'", shell=True, capture_output=True, text=True, diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index da29783..6584ef8 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -110,16 +110,16 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread, woken by each poll tick rather than its own timer -- see OTel_Exporter/DESIGN.md. + # Start background cleanup thread, woken by each poll tick rather than its own timer -- see ventis/OTLP_Exporter/DESIGN.md. self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() - # Spawn the OTLP exporter as a separate process (see OTel_Exporter/DESIGN.md), + # Spawn the OTLP exporter as a separate process (see ventis/OTLP_Exporter/DESIGN.md), # supervised so it gets restarted if it ever exits unexpectedly. otel_exporter_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "OTel_Exporter", + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "OTLP_Exporter", ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() @@ -191,7 +191,12 @@ def _load_config(config_path): project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: - return yaml.safe_load(f) + config = yaml.safe_load(f) + if "ec2" in config: + config["ec2"] = GlobalController._expand_env_value(config["ec2"]) + if "database" in config: + config["database"] = GlobalController._expand_env_value(config["database"]) + return config @staticmethod def _load_dotenv(path): @@ -212,13 +217,13 @@ def _load_dotenv(path): os.environ[key] = value @staticmethod - def _expand_otel_value(value): + def _expand_env_value(value): if isinstance(value, str): return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), value) if isinstance(value, dict): - return {key: GlobalController._expand_otel_value(item) for key, item in value.items()} + return {key: GlobalController._expand_env_value(item) for key, item in value.items()} if isinstance(value, list): - return [GlobalController._expand_otel_value(item) for item in value] + return [GlobalController._expand_env_value(item) for item in value] return value @staticmethod @@ -242,7 +247,7 @@ def _otel_exporter_env(otel_cfg): ) if "destinations" in otel_cfg: - destinations = GlobalController._expand_otel_value(otel_cfg["destinations"]) + destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) for destination in destinations: if destination.get("name") == "langfuse": public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") @@ -639,12 +644,12 @@ def _poll_controllers(self): # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which # terminates every managed process) via the signal handler before this line is # reached -- without the guard, this could respawn a process just intentionally - # killed. See OTel_Exporter/DESIGN.md. + # killed. See ventis/OTLP_Exporter/DESIGN.md. if self.running: self.process_supervisor.check_and_respawn() # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer - # gates every other instance's poll -- see OTel_Exporter/DESIGN.md. + # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. instances = self.instance_manager.list_instances() if instances: with ThreadPoolExecutor(max_workers=len(instances)) as executor: diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 8f5724c..8c061bc 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -3,7 +3,7 @@ register() + start_all() spawn processes; check_and_respawn() (call from GC's existing poll tick) restarts any that exit unexpectedly; terminate_all() (call from GC's shutdown path) stops them all cleanly. Deliberately GC-agnostic -- callers are responsible for not -calling check_and_respawn() during their own shutdown (see OTel_Exporter/DESIGN.md's +calling check_and_respawn() during their own shutdown (see ventis/OTLP_Exporter/DESIGN.md's shutdown-race note). """ diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 803fc2d..4647619 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -17,9 +17,7 @@ import yaml # Packages every agent container needs regardless of its specific business logic. -# grpcio-tools/pyyaml/ipdb/ipython aren't needed/used, but keeping to keep the scope constrained right now -# - Leave a comment if you want me to remove these, I kept them in since you originally had them but they aren't used -BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "ipdb", "ipython", "boto3"] +BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "boto3"] # Workflow will always require these BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask", "sqlalchemy", "psycopg[binary]"] @@ -322,6 +320,21 @@ def _copy_files(output_dir, files_to_copy): shutil.copy2(src, dest_path) +def _write_entrypoint_file(src, dest_path, project_dir): + """Copy an entrypoint file to dest_path, injecting a sys.path entry for its + original sibling directory so a co-located, non-stub helper import still resolves.""" + original_dir = os.path.dirname(os.path.relpath(src, project_dir)) if project_dir else "" + if not original_dir: + shutil.copy2(src, dest_path) + return + injection = ( + f"import sys, os\n" + f"sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), {original_dir!r}))\n" + ) + with open(src) as f, open(dest_path, "w") as out: + out.write(injection + f.read()) + + def generate_docker( yaml_path, agent_file, @@ -405,8 +418,7 @@ def generate_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - - files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -416,6 +428,14 @@ def generate_docker( _copy_files(output_dir, files_to_copy) + # Copy the agent's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(agent_file), + os.path.join(output_dir, os.path.basename(agent_file)), + project_dir, + ) + # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -499,7 +519,6 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ - (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), (os.path.join(script_dir, "deploy.py"), "deploy.py"), @@ -527,6 +546,7 @@ def generate_workflow_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -536,6 +556,14 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) + # Copy the workflow's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(workflow_file), + os.path.join(output_dir, workflow_basename), + project_dir, + ) + # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time From 9bbc35d2206016e72b65061579d7bf2e62098c47 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 28 Aug 2026 13:58:05 -0700 Subject: [PATCH 13/31] rough draft --- .../portfolio/config/global_controller.yaml | 25 +++++---- .../portfolio/workflow/portfolio_workflow.py | 1 - pyproject.toml | 2 +- requirements.txt | 2 - tests/test_otel_exporter_fanout.py | 2 +- tests/test_otel_exporter_fields.py | 2 +- .../OTLP_Exporter}/DESIGN.md | 6 +- ventis/OTLP_Exporter/SCHEMA.md | 56 +++++++++++++++++++ .../OTLP_Exporter}/__init__.py | 0 .../OTLP_Exporter}/convert.py | 24 ++++++-- {OTel_Exporter => ventis/OTLP_Exporter}/db.py | 4 +- .../OTLP_Exporter}/otel_exporter.py | 0 ventis/OTLP_Exporter/otel_queue.db | 0 ventis/cli.py | 21 +++++-- .../cloud_provider_logic/EC2/_runtime.py | 5 +- ventis/controller/global_controller.py | 25 +++++---- ventis/controller/utils/process_supervisor.py | 2 +- ventis/stub_generator.py | 40 +++++++++++-- 18 files changed, 166 insertions(+), 51 deletions(-) rename {OTel_Exporter => ventis/OTLP_Exporter}/DESIGN.md (98%) create mode 100644 ventis/OTLP_Exporter/SCHEMA.md rename {OTel_Exporter => ventis/OTLP_Exporter}/__init__.py (100%) rename {OTel_Exporter => ventis/OTLP_Exporter}/convert.py (72%) rename {OTel_Exporter => ventis/OTLP_Exporter}/db.py (98%) rename {OTel_Exporter => ventis/OTLP_Exporter}/otel_exporter.py (100%) create mode 100644 ventis/OTLP_Exporter/otel_queue.db diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index ab5af4d..edee85f 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -83,18 +83,18 @@ otel: destinations: - name: railway protocol: grpc - endpoint: yamanote.proxy.rlwy.net:19803 + endpoint: ${RAILWAY_OTLP_ENDPOINT} insecure: true headers: {} - - name: langfuse - protocol: http - endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces - headers: {} - name: grafana protocol: http endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} +# - name: langfuse +# protocol: http +# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces +# headers: {} # Polling interval in seconds poll_interval: 5 @@ -107,10 +107,13 @@ redis: # EC2 defaults for `provider: EC2` replicas. ec2: - region: us-east-1 - ami_id: ami-031ff6df47f26b546 - subnet_id: subnet-0638ac6d79d488124 + region: ${EC2_REGION} + ami_id: ${EC2_AMI_ID} + subnet_id: ${EC2_SUBNET_ID} security_group_ids: - - sg-025daf3a98e06cef3 - ssh_user: ubuntu - ssh_private_key_path: ~/.ssh/ventis_ec2 + - ${EC2_SECURITY_GROUP_ID} + ssh_user: ${EC2_SSH_USER} + ssh_private_key_path: ${EC2_SSH_PRIVATE_KEY_PATH} + +database: + url: ${DATABASE_URL} diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 299a3f5..b8b684a 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -46,7 +46,6 @@ def main( advisor = AdvisorAgent() # Stage 0: parse the free-text request into structured holdings + window. - intent = intent_agent.parse(query=query).value() # parse() returns a dict, but a Future's .value() only ever gives back the # raw string ventis stored in Redis -- same deserialization requirement as # every other dict-returning agent call below. diff --git a/pyproject.toml b/pyproject.toml index 9ae1234..40cb43a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["ventis*", "OTel_Exporter*"] +include = ["ventis*"] [tool.setuptools.package-data] ventis = [ diff --git a/requirements.txt b/requirements.txt index dd7a254..f06e0b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,6 @@ grpcio-tools redis pyyaml flask -ipdb -ipython sqlalchemy psycopg[binary] psutil diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index ed1b423..0691d61 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -13,7 +13,7 @@ ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # ``otel_exporter.py`` is also executed as a script from its own directory and # therefore imports ``convert`` and ``db`` as top-level modules. -sys.path.insert(0, os.path.join(ROOT, "OTel_Exporter")) +sys.path.insert(0, os.path.join(ROOT, "ventis", "OTLP_Exporter")) import db # noqa: E402 import otel_exporter # noqa: E402 diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index 4a62c58..ff8cad6 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -5,7 +5,7 @@ import unittest from unittest.mock import patch -from OTel_Exporter import convert, db +from ventis.OTLP_Exporter import convert, db class OTelExporterFieldTests(unittest.TestCase): diff --git a/OTel_Exporter/DESIGN.md b/ventis/OTLP_Exporter/DESIGN.md similarity index 98% rename from OTel_Exporter/DESIGN.md rename to ventis/OTLP_Exporter/DESIGN.md index 86a2a95..287fffd 100644 --- a/OTel_Exporter/DESIGN.md +++ b/ventis/OTLP_Exporter/DESIGN.md @@ -48,7 +48,7 @@ Decisions (final status): Configuration is read at exporter startup; changing it requires a GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own - SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing + SQLite file (`ventis/OTLP_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing `send_runtime_information` write. Keeps this pipeline's schema/state fully decoupled from the dashboard/cost table. @@ -95,7 +95,7 @@ endpoint and headers to the SDK. `BatchSpanProcessor(..., schedule_delay_millis= `max_export_batch_size` is left at the SDK default (512), which already approximates the original "500 spans" batching ask without any override needed. -### 2. `OTel_Exporter/otel_exporter.py` +### 2. `ventis/OTLP_Exporter/otel_exporter.py` A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM stays responsive), calling `_send_pending()` each tick. At startup it constructs one independent OTLP exporter and `BatchSpanProcessor` for each configured destination; @@ -111,7 +111,7 @@ each pair may use a different protocol, endpoint, and headers: since spans are hand-built and handed straight to the processors via `on_end()`. - Every processor is shut down on exit, flushing its pending batch independently. -### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) +### 3. Future row → OTel span conversion (`ventis/OTLP_Exporter/convert.py`) `future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is the one that maps to `trace_id`. Both are `uuid4().hex` (32 hex chars / 16 bytes); OTel `trace_id` is 128-bit (16 bytes, fits directly) and `span_id` is 64-bit (8 bytes, needs diff --git a/ventis/OTLP_Exporter/SCHEMA.md b/ventis/OTLP_Exporter/SCHEMA.md new file mode 100644 index 0000000..38f8664 --- /dev/null +++ b/ventis/OTLP_Exporter/SCHEMA.md @@ -0,0 +1,56 @@ +# OTel span storage schema + +Each emitted OTel span is stored as one `otel_spans` record. This chart also defines a +one-to-one `otel_span_attributes` projection, linked by the same `span_id`, for the +known exporter attributes. The current receiver retains the raw `attributes` JSONB map; +the child table is the relational schema described in `otel_spans_schema.txt`. + +```text +┌───────────────────────────┐ ┌────────────────────────────────┐ +│ otel_spans │ │ otel_span_attributes │ +├───────────────────────────┤ ├────────────────────────────────┤ +│ PK span_id │────1:1───│ PK/FK span_id │ +│ trace_id │ │ model and agent ID │ +│ parent_span_id │ │ CPU and GPU │ +│ name │ │ timing and token usage │ +│ kind │ │ input and output │ +│ start/end time (ns) │ │ project and error count │ +│ status code/message │ │ server/token/total cost │ +│ attributes (JSONB) │ │ cache tokens/hit ratio │ +│ events (JSONB) │ └────────────────────────────────┘ +└───────────────────────────┘ +``` + +## `otel_spans` + +| Column | Meaning | +| --- | --- | +| `span_id` | Unique identifier for this span. | +| `trace_id` | Identifier shared by all spans in the same trace. | +| `parent_span_id` | Parent span; empty for a root span. | +| `name` | Operation name, such as an agent method. | +| `kind` | OTel role of the work; current exporter spans are `SPAN_KIND_INTERNAL`. | +| `start_time_unix_nano` / `end_time_unix_nano` | Raw Unix timestamps in nanoseconds. | +| `status_code` / `status_message` | OTel outcome: normally `STATUS_CODE_UNSET`, or `STATUS_CODE_ERROR` with an error message. | +| `attributes` | Complete raw OTel attribute map (JSONB). | +| `events` | OTel events, including any `exception` event (JSONB). | + +## `otel_span_attributes` + +This one-to-one projection mirrors every attribute currently emitted by +`convert.waiting_row_to_span()`. Fields are nullable because OTel omits an attribute +whose source value is `None`. + +| Group | Columns | +| --- | --- | +| Model and agent | `gen_ai.request.model`, `gen_ai.agent.id` | +| Resources and timing | `cpu`, `gpu`, `execution_time_ms`, `queue_time_ms` | +| Token usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `token_count`, `gen_ai.usage.cache_read.input_tokens` | +| Input and output | `langfuse.observation.input`, `langfuse.observation.output` | +| Project and errors | `project_id`, `error_count` | +| Costs | `server_cost`, `token_cost`, `gen_ai.usage.cost` | +| Cache | `cache_hit_ratio` | + +The DBML source is [`../otel_spans_schema.txt`](../otel_spans_schema.txt). + +Successful spans use `STATUS_CODE_UNSET` rather than `STATUS_CODE_OK` because OpenTelemetry reserves `OK` for application- or operator-validated success, while instrumentation normally sets a status only when it records an error. diff --git a/OTel_Exporter/__init__.py b/ventis/OTLP_Exporter/__init__.py similarity index 100% rename from OTel_Exporter/__init__.py rename to ventis/OTLP_Exporter/__init__.py diff --git a/OTel_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py similarity index 72% rename from OTel_Exporter/convert.py rename to ventis/OTLP_Exporter/convert.py index b7fb493..66da972 100644 --- a/OTel_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -3,7 +3,7 @@ Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects directly instead of going through Tracer.start_span() -- there's no live tracer here, futures already finished (sometimes in another process), so this is a historical-row -conversion, not live tracing. See OTel_Exporter/DESIGN.md for why this deviates from +conversion, not live tracing. See ventis/OTLP_Exporter/DESIGN.md for why this deviates from the SDK's usual advice against constructing ReadableSpan by hand. """ @@ -64,9 +64,17 @@ def waiting_row_to_span(row): ) status = Status(StatusCode.ERROR, description=row.get("error_message")) - # Model and token usage use OTel GenAI semantic-convention names. Observation - # input/output use Langfuse's documented JSON-string attributes. The remaining - # Ventis infrastructure values have no GenAI equivalent, so they keep plain names. + # Model, token, agent, and cache-read usage use OTel GenAI semantic-convention + # names (see https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/). + # total_cost uses gen_ai.usage.cost, which isn't yet an official semconv attribute + # but is the name Langfuse's OTel ingestion actually reads (its JSON cost_details + # attribute is currently broken -- see langfuse/langfuse#11030). Observation + # input/output use Langfuse's documented JSON-string attributes. `errors` is named + # error_count, not "errors"/"error", to avoid colliding with OTel's reserved + # error.* namespace (error.type etc.), which describes a single error, not a + # count. The remaining Ventis-specific values (project_id, server/token cost + # breakdown, cache_hit_ratio) have no GenAI or Langfuse equivalent, so they keep + # plain names. attributes = { k: v for k, v in { @@ -80,6 +88,14 @@ def waiting_row_to_span(row): "token_count": row.get("token_count"), "langfuse.observation.input": row.get("input"), "langfuse.observation.output": row.get("output"), + "project_id": row.get("project_id"), + "gen_ai.agent.id": row.get("agent_id"), + "error_count": row.get("errors"), + "server_cost": row.get("server_cost"), + "token_cost": row.get("token_cost"), + "gen_ai.usage.cost": row.get("total_cost"), + "gen_ai.usage.cache_read.input_tokens": row.get("cached_tokens"), + "cache_hit_ratio": row.get("cache_hit_ratio"), }.items() if v is not None } diff --git a/OTel_Exporter/db.py b/ventis/OTLP_Exporter/db.py similarity index 98% rename from OTel_Exporter/db.py rename to ventis/OTLP_Exporter/db.py index a8a675f..e6a1834 100644 --- a/OTel_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -130,7 +130,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH if not fid or not session_id: continue agent_id = raw.get("agent") - started_at = float(raw.get("created_at") or 0) or None + started_at = float(raw.get("created_at") or 0) finished_at = float(raw["finished_at"]) if raw.get("finished_at") else None execution_time_ms = ( round((finished_at - started_at) * 1000) @@ -160,7 +160,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH * _TOKEN_COST_MULTIPLIER ) # Server cost needs an elapsed duration -- only available once finished. - if finished_at and started_at: + if finished_at is not None: server_cost = ( pricing.compute_server_cost( redis_client.get(f"agent:{agent_id}:instance_type") diff --git a/OTel_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py similarity index 100% rename from OTel_Exporter/otel_exporter.py rename to ventis/OTLP_Exporter/otel_exporter.py diff --git a/ventis/OTLP_Exporter/otel_queue.db b/ventis/OTLP_Exporter/otel_queue.db new file mode 100644 index 0000000..e69de29 diff --git a/ventis/cli.py b/ventis/cli.py index 920df15..31a32e0 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -238,6 +238,15 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path + # Maps each generated stub's basename to its agent's entrypoint path, so a + # stub can also be placed at its nested, entrypoint-mirrored location. + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -302,9 +311,9 @@ def cmd_build(args): api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # A workflow script imports stubs by flat module name (e.g. `from - # intent_agent import ...`), not by the agent's own entrypoint path. - stub_entrypoints=None, + # Stubs are placed both flat and at their entrypoint-mirrored path, + # so both flat and nested import styles resolve to the stub. + stub_entrypoints=stub_entrypoints, ) else: @@ -340,9 +349,9 @@ def cmd_build(args): stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # Same reasoning as the workflow call above: this project's agents - # import each other's stubs by flat module name, not entrypoint path. - stub_entrypoints=None, + # Same reasoning as the workflow call above: stubs are placed both + # flat and at their entrypoint-mirrored path. + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 9955fa2..7e4e9ab 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -239,11 +239,12 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, logger.info("Transferring image %s to %s", image, host) result = subprocess.run( "set -o pipefail; " - f"docker save {shlex.quote(image)} | ssh -o StrictHostKeyChecking=no " + f"docker save {shlex.quote(image)} | zstd -T0 | ssh -o StrictHostKeyChecking=no " f"-o IdentitiesOnly=yes -o ConnectTimeout=10 " f"-o ServerAliveInterval=10 -o ServerAliveCountMax=3 " f"-i {shlex.quote(key)} " - f"{shlex.quote(f'{ssh_user}@{host}')} 'sudo docker load'", + f"{shlex.quote(f'{ssh_user}@{host}')} " + "'set -o pipefail; zstd -d | sudo docker load'", shell=True, capture_output=True, text=True, diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index bdec683..16e0e9c 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -110,16 +110,16 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread, woken by each poll tick rather than its own timer -- see OTel_Exporter/DESIGN.md. + # Start background cleanup thread, woken by each poll tick rather than its own timer -- see ventis/OTLP_Exporter/DESIGN.md. self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() - # Spawn the OTLP exporter as a separate process (see OTel_Exporter/DESIGN.md), + # Spawn the OTLP exporter as a separate process (see ventis/OTLP_Exporter/DESIGN.md), # supervised so it gets restarted if it ever exits unexpectedly. otel_exporter_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "OTel_Exporter", + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "OTLP_Exporter", ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() @@ -191,7 +191,12 @@ def _load_config(config_path): project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: - return yaml.safe_load(f) + config = yaml.safe_load(f) + if "ec2" in config: + config["ec2"] = GlobalController._expand_env_value(config["ec2"]) + if "database" in config: + config["database"] = GlobalController._expand_env_value(config["database"]) + return config @staticmethod def _load_dotenv(path): @@ -212,13 +217,13 @@ def _load_dotenv(path): os.environ[key] = value @staticmethod - def _expand_otel_value(value): + def _expand_env_value(value): if isinstance(value, str): return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), value) if isinstance(value, dict): - return {key: GlobalController._expand_otel_value(item) for key, item in value.items()} + return {key: GlobalController._expand_env_value(item) for key, item in value.items()} if isinstance(value, list): - return [GlobalController._expand_otel_value(item) for item in value] + return [GlobalController._expand_env_value(item) for item in value] return value @staticmethod @@ -242,7 +247,7 @@ def _otel_exporter_env(otel_cfg): ) if "destinations" in otel_cfg: - destinations = GlobalController._expand_otel_value(otel_cfg["destinations"]) + destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) for destination in destinations: if destination.get("name") == "langfuse": public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") @@ -639,7 +644,7 @@ def _poll_controllers(self): # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which # terminates every managed process) via the signal handler before this line is # reached -- without the guard, this could respawn a process just intentionally - # killed. See OTel_Exporter/DESIGN.md. + # killed. See ventis/OTLP_Exporter/DESIGN.md. if self.running: self.process_supervisor.check_and_respawn() diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 8f5724c..8c061bc 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -3,7 +3,7 @@ register() + start_all() spawn processes; check_and_respawn() (call from GC's existing poll tick) restarts any that exit unexpectedly; terminate_all() (call from GC's shutdown path) stops them all cleanly. Deliberately GC-agnostic -- callers are responsible for not -calling check_and_respawn() during their own shutdown (see OTel_Exporter/DESIGN.md's +calling check_and_respawn() during their own shutdown (see ventis/OTLP_Exporter/DESIGN.md's shutdown-race note). """ diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 803fc2d..4647619 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -17,9 +17,7 @@ import yaml # Packages every agent container needs regardless of its specific business logic. -# grpcio-tools/pyyaml/ipdb/ipython aren't needed/used, but keeping to keep the scope constrained right now -# - Leave a comment if you want me to remove these, I kept them in since you originally had them but they aren't used -BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "ipdb", "ipython", "boto3"] +BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "boto3"] # Workflow will always require these BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask", "sqlalchemy", "psycopg[binary]"] @@ -322,6 +320,21 @@ def _copy_files(output_dir, files_to_copy): shutil.copy2(src, dest_path) +def _write_entrypoint_file(src, dest_path, project_dir): + """Copy an entrypoint file to dest_path, injecting a sys.path entry for its + original sibling directory so a co-located, non-stub helper import still resolves.""" + original_dir = os.path.dirname(os.path.relpath(src, project_dir)) if project_dir else "" + if not original_dir: + shutil.copy2(src, dest_path) + return + injection = ( + f"import sys, os\n" + f"sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), {original_dir!r}))\n" + ) + with open(src) as f, open(dest_path, "w") as out: + out.write(injection + f.read()) + + def generate_docker( yaml_path, agent_file, @@ -405,8 +418,7 @@ def generate_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - - files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -416,6 +428,14 @@ def generate_docker( _copy_files(output_dir, files_to_copy) + # Copy the agent's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(agent_file), + os.path.join(output_dir, os.path.basename(agent_file)), + project_dir, + ) + # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -499,7 +519,6 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ - (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), (os.path.join(script_dir, "deploy.py"), "deploy.py"), @@ -527,6 +546,7 @@ def generate_workflow_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -536,6 +556,14 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) + # Copy the workflow's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(workflow_file), + os.path.join(output_dir, workflow_basename), + project_dir, + ) + # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time From 8baed6c324536bc88c0446b16376782c5b5cd9e4 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 11:23:57 -0700 Subject: [PATCH 14/31] Parallelize per-instance polling in _poll_controllers Metrics/telemetry latency scaled with instance count x per-instance round-trip time since every instance was polled sequentially, one blocking the next, with the following tick only starting after the whole pass finished. Extracted the per-instance body into _poll_one_instance (whole body wrapped in one top-level try/except, since ThreadPoolExecutor.map() re-raises on first exception when results are consumed) and run all instances concurrently via the same ThreadPoolExecutor pattern _trigger_cleanup already used. Co-Authored-By: Claude Sonnet 5 --- ventis/controller/global_controller.py | 217 +++++++++++++------------ 1 file changed, 115 insertions(+), 102 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 16e0e9c..6584ef8 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -648,118 +648,131 @@ def _poll_controllers(self): if self.running: self.process_supervisor.check_and_respawn() - for instance in self.instance_manager.list_instances(): + # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer + # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. + instances = self.instance_manager.list_instances() + if instances: + with ThreadPoolExecutor(max_workers=len(instances)) as executor: + list(executor.map(self._poll_one_instance, instances)) + + def _poll_one_instance(self, instance): + """Poll and persist one instance's runtime/metrics/health data; never raises.""" + try: name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, + except Exception as e: + logger.warning("Failed to poll instance %s: %s", instance, e) + return + + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" + + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now - - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, - ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) - else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status + else: + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From cd3a5214099e3bdb4493f26820cafa684b8af2fd Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 13:58:37 -0700 Subject: [PATCH 15/31] cleaned up OTel Exporter --- .../portfolio/config/global_controller.yaml | 4 - tests/test_otel_exporter_fanout.py | 58 +-- tests/test_otel_exporter_fields.py | 36 +- ventis/OTLP_Exporter/DESIGN.md | 73 ++-- ventis/OTLP_Exporter/convert.py | 17 +- ventis/OTLP_Exporter/db.py | 61 +-- ventis/OTLP_Exporter/otel_exporter.py | 67 +-- ventis/controller/global_controller.py | 385 ++++++------------ 8 files changed, 216 insertions(+), 485 deletions(-) diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index edee85f..96f371c 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -91,10 +91,6 @@ otel: endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} -# - name: langfuse -# protocol: http -# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces -# headers: {} # Polling interval in seconds poll_interval: 5 diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 0691d61..96ca1b5 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -110,22 +110,10 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): ], ) - def test_build_processors_preserves_legacy_single_destination_fallback(self): - http_exporter = object() - processor = MagicMock(name="legacy_processor") - with patch.dict( - os.environ, - {"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, - clear=True, - ), patch.object( - otel_exporter, "HttpOTLPSpanExporter", return_value=http_exporter - ) as constructor, patch.object( - otel_exporter, "BatchSpanProcessor", return_value=processor - ): - result = otel_exporter._build_processors() - - self.assertEqual(result, [("legacy", processor)]) - constructor.assert_called_once_with() + def test_build_processors_raises_when_destinations_env_unset(self): + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): + otel_exporter._build_processors() def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): invalid_values = [ @@ -184,46 +172,20 @@ def test_controller_expands_env_and_builds_langfuse_basic_auth(self): ) self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") - def test_controller_env_serializes_destinations_and_keeps_legacy_mapping(self): + def test_controller_env_serializes_destinations_only(self): # Importing the controller is intentionally local: this test remains # runnable in the exporter-only environment used by the focused suite. from ventis.controller.global_controller import GlobalController destinations = self._destination_config() - env = GlobalController._otel_exporter_env( - { - "protocol": "grpc", - "endpoint": "legacy.example:4317", - "headers": {"x-tenant": "demo"}, - "destinations": destinations, - } - ) - self.assertEqual(env["OTEL_EXPORTER_OTLP_PROTOCOL"], "grpc") - self.assertEqual(env["OTEL_EXPORTER_OTLP_ENDPOINT"], "legacy.example:4317") - self.assertEqual(env["OTEL_EXPORTER_OTLP_HEADERS"], "x-tenant=demo") + env = GlobalController._otel_exporter_env({"destinations": destinations}) + self.assertEqual(set(env), {otel_exporter.DESTINATIONS_ENV}) self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) - def test_controller_rejects_invalid_destinations_before_starting_child(self): + def test_controller_env_is_none_when_otel_not_configured(self): from ventis.controller.global_controller import GlobalController - invalid_destinations = [ - [], - [{"name": "railway", "protocol": "grpc"}], - [ - {"name": "same", "protocol": "grpc", "endpoint": "one:4317"}, - { - "name": "same", - "protocol": "http/protobuf", - "endpoint": "https://two", - }, - ], - [{"name": "bad", "protocol": "smtp", "endpoint": "example"}], - ] - for destinations in invalid_destinations: - with self.subTest(destinations=destinations), self.assertRaises(ValueError): - GlobalController._otel_exporter_env( - {"destinations": destinations} - ) + self.assertIsNone(GlobalController._otel_exporter_env({})) def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) @@ -258,7 +220,6 @@ def test_send_pending_delivers_the_same_span_to_every_processor(self): otel_exporter.db, "mark_sent" ) as mark_sent: otel_exporter._processors = [("railway", first), ("langfuse", second)] - otel_exporter._processor = None otel_exporter._send_pending() first.on_end.assert_called_once() @@ -275,7 +236,6 @@ def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_fai otel_exporter.db, "mark_sent" ) as mark_sent: otel_exporter._processors = [("railway", failed), ("langfuse", remaining)] - otel_exporter._processor = None otel_exporter._send_pending() failed.on_end.assert_called_once() diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index ff8cad6..b74177d 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -17,12 +17,7 @@ def setUp(self): def tearDown(self): os.unlink(self.db_path) - def test_init_db_migrates_existing_waiting_table(self): - with sqlite3.connect(self.db_path) as conn: - conn.execute( - "CREATE TABLE waiting (future_id TEXT PRIMARY KEY, session_id TEXT NOT NULL)" - ) - + def test_init_db_creates_waiting_table_with_full_schema(self): db.init_db(self.db_path) with sqlite3.connect(self.db_path) as conn: @@ -63,17 +58,8 @@ def test_fields_are_normalized_and_added_to_span(self): span.attributes["langfuse.observation.output"], row["output"] ) - def test_split_hash_result_is_loaded_for_finished_rows(self): - class SplitHashRedis: - def hget(self, key, field): - self.request = (key, field) - return '{"recommendation": "hold"}' - - def get(self, key): - return None - + def test_error_message_is_wired_from_redis_error_field(self): db.init_db(self.db_path) - redis = SplitHashRedis() raw = { "future_id": "11112222333344445555666677778888", "request_id": "88887777666655554444333322221111", @@ -83,16 +69,24 @@ def get(self, key): "result": "", "created_at": "1.0", "finished_at": "2.0", - "failed": "0", + "failed": "1", + "error": "agent exploded", } with patch.object(db.pricing, "compute_token_cost", return_value=0.0): - db.write_waiting_rows([raw], redis_client=redis, db_path=self.db_path) + db.write_waiting_rows([raw], db_path=self.db_path) with sqlite3.connect(self.db_path) as conn: - output = conn.execute("SELECT output FROM waiting").fetchone()[0] - self.assertEqual(redis.request, (f"future:{raw['future_id']}", "result")) - self.assertEqual(json.loads(output), {"recommendation": "hold"}) + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM waiting").fetchone() + + self.assertEqual(row["error_message"], "agent exploded") + + span = convert.waiting_row_to_span(row) + self.assertEqual(span.status.description, "agent exploded") + self.assertEqual( + span.events[0].attributes["exception.message"], "agent exploded" + ) if __name__ == "__main__": diff --git a/ventis/OTLP_Exporter/DESIGN.md b/ventis/OTLP_Exporter/DESIGN.md index d433fe0..ec2957d 100644 --- a/ventis/OTLP_Exporter/DESIGN.md +++ b/ventis/OTLP_Exporter/DESIGN.md @@ -27,26 +27,22 @@ Decisions (final status): isolation from GC's core polling/health loop and independent restart, at low added complexity since SQLite is already the entire hand-off boundary between the two. - **Config**: implemented via a new `otel:` section in `global_controller.yaml` - (`protocol`/`endpoint`/`headers`), *not* by making `otel_exporter.py` itself - config-aware. `GlobalController` translates that section into the OTel SDK's own - standard env vars (`OTEL_EXPORTER_OTLP_PROTOCOL`/`_ENDPOINT`/`_HEADERS`) and passes - them to the exporter subprocess via `ProcessSupervisor.register(..., env=...)`. The - exporter still just constructs `OTLPSpanExporter()` with no explicit args (endpoint - and headers are resolved by the SDK itself from those env vars, same as always) and - reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly, to pick the gRPC vs HTTP exporter - class — the one piece of protocol selection the plain SDK classes don't do on their - own. Deliberately vendor-neutral: no backend name (Postgres, Langfuse, or otherwise) - appears anywhere in `otel_exporter.py`; the destination is 100% deploy-time config, - set once in `global_controller.yaml` and never touched by app code again. The - originally-planned `database.url` repurposing (below, kept for history) was decided - against — env-var configuration is the SDK's own idiomatic mechanism, so no - exporter-side config plumbing was added, only a GC-side YAML→env-var translation. - The initial multi-destination extension uses one `otel.destinations` list and one - independent exporter/`BatchSpanProcessor` pair per destination. gRPC and HTTP - destinations may be mixed in the same list. The legacy single-destination fields - remain supported through the original standard-environment-variable path. - Configuration is read at exporter startup; changing it requires a - GlobalController/exporter restart. + holding a `destinations` list, *not* by making `otel_exporter.py` itself + config-aware. `GlobalController` serializes that list to JSON and passes it to the + exporter subprocess as a single `VENTIS_OTEL_DESTINATIONS` env var via + `ProcessSupervisor.register(..., env=...)`. The exporter builds one independent + exporter/`BatchSpanProcessor` pair per destination, picking the gRPC vs HTTP + exporter class from each destination's `protocol` field. gRPC and HTTP destinations + may be mixed in the same list. Deliberately vendor-neutral: no backend name + (Postgres, Langfuse, or otherwise) appears anywhere in `otel_exporter.py`; the + destination is 100% deploy-time config, set once in `global_controller.yaml` and + never touched by app code again. The originally-planned `database.url` repurposing + (below, kept for history) was decided against — env-var configuration is the SDK's + own idiomatic mechanism, so no exporter-side config plumbing was added, only a + GC-side YAML→env-var translation. If `otel.destinations` is absent, GlobalController + logs that no OTel metrics collection will happen and skips starting the exporter + subprocess entirely. Configuration is read at exporter startup; changing it requires + a GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own SQLite file (`ventis/OTLP_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing @@ -77,16 +73,19 @@ otel: - name: langfuse protocol: http endpoint: https://cloud.langfuse.com/api/public/otel/v1/traces - headers: {} # e.g. Authorization: "Basic " + headers: + Authorization: Basic ${LANGFUSE_OTLP_HEADERS} # deployer pre-encodes public:secret ``` -`GlobalController._otel_exporter_env()` translates each destination into the exporter -process's destination configuration and hands it to `ProcessSupervisor.register( +`GlobalController._otel_exporter_env()` translates the `destinations` list into +`VENTIS_OTEL_DESTINATIONS` and hands it to `ProcessSupervisor.register( "otel_exporter", ..., env=...)`, which supports an `env` param (merged on top of the -parent process's own environment, not a replacement). The legacy single-destination -`protocol`/`endpoint`/`headers` form remains valid and continues through the SDK's -standard OTLP environment variables. Omitting `otel:` entirely falls back to whatever -ambient env the exporter subprocess would otherwise inherit, same as before this -change. +parent process's own environment, not a replacement). If `otel.destinations` is +absent, `_otel_exporter_env()` returns `None` and `GlobalController.__init__` skips +registering the exporter subprocess entirely, logging that no OTel metrics +collection will happen. No shape +validation is duplicated on the GlobalController side (deliberately: keep this side +simple, `otel_exporter.py` itself validates destination shape at subprocess startup, +and raises if invoked directly without `VENTIS_OTEL_DESTINATIONS` set). `otel_exporter.py` parses the destination configuration at startup and constructs the appropriate OTLP exporter for each entry (gRPC or HTTP), passing that destination's @@ -167,21 +166,7 @@ Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall the poll loop's health checks and OTel writes. -### 6. Parallelized per-instance polling (`ventis/controller/global_controller.py`) -`_poll_controllers` used to loop over every instance sequentially -- Redis reads, an -OTel sqlite write, and up to two Postgres writes per instance, one instance fully -blocking the next, with the following poll tick only starting after the whole pass -finished. Total metrics/telemetry latency scaled with instance count x round-trip -time, not the configured `poll_interval`. Fixed by extracting the per-instance body -into `_poll_one_instance` (its whole body wrapped in one top-level try/except, since -`ThreadPoolExecutor.map()` re-raises on first exception when results are consumed) -and running all instances concurrently via the same `ThreadPoolExecutor` pattern -`_trigger_cleanup` already used. Known, pre-existing, previously acknowledged in -commit `a6694d9`'s own message but never actually fixed (a same-named follow-up -branch was found to contain no real threading changes) -- see company-memory for -the investigation. - -### 7. Dependencies (all added) +### 6. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). @@ -189,8 +174,6 @@ config work, since `protocol: http` now needs that package importable). ## Known gaps (not yet built) - `WriteResult()` passes an undefined `error_message` variable to its fan-out callback, which can interrupt remote consumer propagation after the callback hash is persisted. -- Redis records failure text under `error`, but the waiting-table writer reads - `error_name`/`error_message`, so exported exception details are usually empty. - Rows are marked `sent` immediately after `BatchSpanProcessor.on_end()` accepts them, before the asynchronous OTLP export is confirmed; a later delivery failure can lose a span while leaving `sent = 1`. diff --git a/ventis/OTLP_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py index 66da972..5e6ac74 100644 --- a/ventis/OTLP_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -1,10 +1,7 @@ -"""Convert a `waiting` table row (see db.py) into an OTel ReadableSpan. +"""Converts a future into an OTel ReadableSpan. -Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects -directly instead of going through Tracer.start_span() -- there's no live tracer here, -futures already finished (sometimes in another process), so this is a historical-row -conversion, not live tracing. See ventis/OTLP_Exporter/DESIGN.md for why this deviates from -the SDK's usual advice against constructing ReadableSpan by hand. +Pure function, no I/O, no batching, no network calls. Futures already finished, so this is just a +conversion. """ from opentelemetry.sdk.trace import EXCEPTION_MESSAGE, EXCEPTION_TYPE, Event, ReadableSpan @@ -66,13 +63,7 @@ def waiting_row_to_span(row): # Model, token, agent, and cache-read usage use OTel GenAI semantic-convention # names (see https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/). - # total_cost uses gen_ai.usage.cost, which isn't yet an official semconv attribute - # but is the name Langfuse's OTel ingestion actually reads (its JSON cost_details - # attribute is currently broken -- see langfuse/langfuse#11030). Observation - # input/output use Langfuse's documented JSON-string attributes. `errors` is named - # error_count, not "errors"/"error", to avoid colliding with OTel's reserved - # error.* namespace (error.type etc.), which describes a single error, not a - # count. The remaining Ventis-specific values (project_id, server/token cost + # total_cost uses gen_ai.usage.cost. The remaining Ventis-specific values (project_id, server/token cost # breakdown, cache_hit_ratio) have no GenAI or Langfuse equivalent, so they keep # plain names. attributes = { diff --git a/ventis/OTLP_Exporter/db.py b/ventis/OTLP_Exporter/db.py index e6a1834..005ba71 100644 --- a/ventis/OTLP_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -12,21 +12,18 @@ import os import sqlite3 -from ventis.controller.utils import pricing +from ventis.controller.utils import pricing +# Will need to eventually delete dependency on this and move to OTLP +# It is currently stored here for backcompat with the old telemetry collecting + DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "otel_queue.db") -# Demo-only multipliers for scaling displayed costs; not real recorded costs. Kept -# deliberately standalone/duplicated from telemetry_logging.py's identical constants -# (rather than importing them) so this module has no dependency on it -- keep these in -# sync by hand if the multipliers there ever change. +# Demo-only multipliers for scaling displayed costs, DELETE FOR MORE ACCURATE METRICS _TOKEN_COST_MULTIPLIER = 10000 _SERVER_COST_MULTIPLIER = 100000 -# Timestamps are stored as unix epoch seconds (matching the Redis future hash fields -# they're read from), not as SQLite datetime strings. Column set mirrors -# runtime_information 1:1 (see telemetry_logging.py) plus this pipeline's own additions -# (error_name/error_message/sent). +# Table schema _TABLE_COLUMNS = """ future_id TEXT PRIMARY KEY, parent_id TEXT, @@ -55,28 +52,14 @@ name TEXT, input TEXT, output TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, sent BOOLEAN DEFAULT 0 """ -_MIGRATION_COLUMNS = { - "name": "TEXT", - "input": "TEXT", - "output": "TEXT", -} - - def init_db(db_path=DB_PATH): - """Create the waiting table and add columns missing from older databases.""" + """Create the waiting table if it doesn't already exist.""" conn = sqlite3.connect(db_path) try: conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") - existing_columns = { - row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() - } - for column, column_type in _MIGRATION_COLUMNS.items(): - if column not in existing_columns: - conn.execute(f"ALTER TABLE waiting ADD COLUMN {column} {column_type}") conn.commit() finally: conn.close() @@ -147,20 +130,16 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH part for part in (service, method) if part ) result = raw.get("result") - # Compatibility with pre-consolidation deployments, where completion - # metrics live in future:{id}:metrics but result lives in future:{id}. - # Unified hashes already include result and avoid this extra read. - if not result and finished_at and redis_client is not None: - result = redis_client.hget(f"future:{fid}", "result") - - token_cost = ( - pricing.compute_token_cost( - raw.get("model"), input_token_count, output_token_count - ) - * _TOKEN_COST_MULTIPLIER - ) - # Server cost needs an elapsed duration -- only available once finished. + + # Cost figures are only meaningful once the future has finished, so skip + # computing them until then rather than recomputing on every poll. if finished_at is not None: + token_cost = ( + pricing.compute_token_cost( + raw.get("model"), input_token_count, output_token_count + ) + * _TOKEN_COST_MULTIPLIER + ) server_cost = ( pricing.compute_server_cost( redis_client.get(f"agent:{agent_id}:instance_type") @@ -171,6 +150,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH * _SERVER_COST_MULTIPLIER ) else: + token_cost = 0.0 server_cost = 0.0 conn.execute( @@ -203,7 +183,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH "cached_tokens": cached_tokens, "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, "error_name": raw.get("error_name"), - "error_message": raw.get("error_message"), + "error_message": raw.get("error") or raw.get("error_message"), "name": name or agent_id or "unknown_agent", "input": _normalize_json_text(raw.get("args")), "output": _normalize_json_text(result), @@ -215,10 +195,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH def mark_sent(future_id, db_path=DB_PATH): - """Mark one waiting row sent. Call this immediately after successfully handing its - span to the batch processor -- one row, one commit -- so a crash between two rows' - sends can't leave an already-sent row unmarked (which would cause a duplicate send - on the next run).""" + """Mark one waiting row sent. Atomic Operation""" conn = sqlite3.connect(db_path) try: conn.execute("UPDATE waiting SET sent = 1 WHERE future_id = ?", (future_id,)) diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index cad2af8..eafb786 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,11 +5,8 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController may provide a JSON list in ``VENTIS_OTEL_DESTINATIONS``. That is a -Ventis-specific configuration because the standard OTEL exporter environment -variables describe only one destination. If it is absent, the original single -destination behavior is retained: the exporter class and its settings are selected -from the standard OTEL environment variables and SDK defaults. +GlobalController provides a JSON list in ``VENTIS_OTEL_DESTINATIONS``, required because +the standard OTEL exporter environment variables describe only one destination. """ import json @@ -35,33 +32,11 @@ logger = logging.getLogger(__name__) _running = True -_processor = None _processors = [] POLL_INTERVAL_SECONDS = 5 DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" -def _normalize_protocol(protocol): - """Return the exporter family for a configured protocol name.""" - if not isinstance(protocol, str) or not protocol.strip(): - raise ValueError("destination protocol must be a non-empty string") - normalized = protocol.strip().lower().replace("_", "-") - if normalized in {"grpc", "otlp/grpc", "grpc/protobuf", "grpc-protobuf"}: - return "grpc" - if normalized in { - "http", - "http/protobuf", - "http-protobuf", - "http/proto", - "http+protobuf", - "protobuf", - }: - return "http" - raise ValueError( - f"unsupported destination protocol {protocol!r}; expected grpc or http/protobuf" - ) - - def _validate_destination(destination, index): if not isinstance(destination, dict): raise ValueError(f"destination {index} must be an object") @@ -70,7 +45,7 @@ def _validate_destination(destination, index): if not isinstance(name, str) or not name.strip(): raise ValueError(f"destination {index} name must be a non-empty string") - protocol = _normalize_protocol(destination.get("protocol")) + protocol = destination.get("protocol") # must be exactly "grpc" or "http" endpoint = destination.get("endpoint") if not isinstance(endpoint, str) or not endpoint.strip(): raise ValueError(f"destination {name!r} endpoint must be a non-empty string") @@ -112,13 +87,7 @@ def _validate_destination(destination, index): def _configured_destinations(): - """Parse and validate the Ventis multi-destination environment variable. - - ``None`` means no Ventis-specific configuration was supplied, so callers can - preserve legacy OTEL environment-variable behavior. An empty or malformed value - is an explicit configuration error and fails startup rather than silently - exporting to the wrong destination. - """ + """Parse and validate the Ventis multi-destination environment variable.""" raw = os.environ.get(DESTINATIONS_ENV) if raw is None: return None @@ -142,18 +111,15 @@ def _configured_destinations(): def _build_exporter(destination): - """Construct one explicitly configured exporter without logging credentials.""" + """Construct one OTLP exporter.""" kwargs = { "endpoint": destination["endpoint"], } - if destination["headers"] is not None: - kwargs["headers"] = destination["headers"] - if destination["timeout"] is not None: - kwargs["timeout"] = destination["timeout"] + if destination["headers"] is not None: kwargs["headers"] = destination["headers"] # fmt: skip + if destination["timeout"] is not None: kwargs["timeout"] = destination["timeout"] # fmt: skip if destination["protocol"] == "grpc": - if destination["insecure"] is not None: - kwargs["insecure"] = destination["insecure"] + if destination["insecure"] is not None: kwargs["insecure"] = destination["insecure"] # fmt: skip return GrpcOTLPSpanExporter(**kwargs) if destination["insecure"] is not None: @@ -166,14 +132,10 @@ def _build_exporter(destination): def _build_processors(): - """Build destination processors, or one legacy processor when unconfigured.""" + """Build one exporter/BatchSpanProcessor pair per configured destination.""" destinations = _configured_destinations() if destinations is None: - protocol = os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower() - exporter_class = ( - HttpOTLPSpanExporter if protocol.startswith("http") else GrpcOTLPSpanExporter - ) - return [("legacy", BatchSpanProcessor(exporter_class(), schedule_delay_millis=1000))] + raise RuntimeError(f"{DESTINATIONS_ENV} is not set; otel.destinations is required") processors = [] try: @@ -205,10 +167,6 @@ def _handle_shutdown(signum, frame): def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" processors = _processors - if not processors and _processor is not None: - # Compatibility for callers that configured the pre-fan-out singular - # ``_processor`` directly (the normal startup path always populates both). - processors = [("legacy", _processor)] if not processors: raise RuntimeError("OTel exporter has no configured processors") @@ -256,14 +214,11 @@ def _send_pending(): def main(): - global _processor, _processors + global _processors signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() _processors = _build_processors() - # Keep the old singular module variable available to integrations that imported - # it, while all sending uses the destination-aware collection above. - _processor = _processors[0][1] logger.info("OTel exporter process started with %d destination(s).", len(_processors)) try: last_poll = 0 diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 6584ef8..ba53881 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,11 +3,8 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit -import base64 -import importlib.util import json import logging -import math import os import re import shlex @@ -19,6 +16,7 @@ from concurrent.futures import ThreadPoolExecutor import yaml +from ventis.OTLP_Exporter import db as otel_db from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs from ventis.controller.utils.env_file import resolve_env_file @@ -110,7 +108,7 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread, woken by each poll tick rather than its own timer -- see ventis/OTLP_Exporter/DESIGN.md. + # Start background cleanup thread self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() @@ -123,21 +121,19 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # Legacy `otel:` fields map straight to the OTel SDK's own standard env vars. - # A `destinations` list is additionally passed as one Ventis-specific JSON - # variable; the exporter subprocess remains a plain OTel process otherwise. + + # Passing OTel info from yaml file to process, so process doesn't have external facing logic otel_env = self._otel_exporter_env(self.config.get("otel", {})) - self.process_supervisor.register( - "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env - ) + if otel_env is not None: + self.process_supervisor.register( + "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + ) + else: + logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") # Initialize/migrate the waiting table synchronously before either the GC or # exporter process can access it. - otel_db_spec = importlib.util.spec_from_file_location( - "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") - ) - self._otel_db = importlib.util.module_from_spec(otel_db_spec) - otel_db_spec.loader.exec_module(self._otel_db) + self._otel_db = otel_db self._otel_db.init_db() self.process_supervisor.start_all() @@ -192,10 +188,7 @@ def _load_config(config_path): GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: config = yaml.safe_load(f) - if "ec2" in config: - config["ec2"] = GlobalController._expand_env_value(config["ec2"]) - if "database" in config: - config["database"] = GlobalController._expand_env_value(config["database"]) + config = GlobalController._expand_env_value(config) return config @staticmethod @@ -228,125 +221,21 @@ def _expand_env_value(value): @staticmethod def _otel_exporter_env(otel_cfg): - """Translate global_controller.yaml's `otel:` section into standard OTLP env - vars for the exporter subprocess. When present, ``destinations`` is passed as - JSON for the exporter to construct a fan-out. Returns None if `otel:` is - absent/empty so the subprocess falls back to the SDK's own defaults untouched. - - The legacy protocol/endpoint/headers mappings intentionally remain unchanged - for existing configurations. - """ - env = {} - if otel_cfg.get("protocol"): - env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] - if otel_cfg.get("endpoint"): - env["OTEL_EXPORTER_OTLP_ENDPOINT"] = otel_cfg["endpoint"] - if otel_cfg.get("headers"): - env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( - f"{k}={v}" for k, v in otel_cfg["headers"].items() - ) - - if "destinations" in otel_cfg: - destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) - for destination in destinations: - if destination.get("name") == "langfuse": - public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") - secret_key = os.environ.get("LANGFUSE_SECRET_KEY") - if public_key and secret_key: - auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode() - destination.setdefault("headers", {})["Authorization"] = f"Basic {auth}" - GlobalController._validate_otel_destinations(destinations) - try: - env["VENTIS_OTEL_DESTINATIONS"] = json.dumps(destinations) - except (TypeError, ValueError) as exc: - # Do not include the offending value: destination configs commonly - # contain credentials in headers. - raise ValueError( - "otel.destinations must contain JSON-serializable values" - ) from exc - return env or None - - @staticmethod - def _validate_otel_destinations(destinations): - """Validate the shape of the optional exporter fan-out configuration. - - Keep this validation deliberately structural: destination-specific options - are interpreted by the exporter. Error messages identify only the location - and type, never destination contents or header values. + """Translate global_controller.yaml's `otel:` section into the exporter + subprocess's env. Returns None if `otel.destinations` is absent, so the + caller skips starting the exporter subprocess entirely. Destination + shape/protocol is validated by the exporter subprocess itself + (otel_exporter.py), not duplicated here. """ - if not isinstance(destinations, list): - raise ValueError("otel.destinations must be a list") - if not destinations: - raise ValueError("otel.destinations must not be empty") - - names = set() - for index, destination in enumerate(destinations): - if not isinstance(destination, dict): - raise ValueError( - f"otel.destinations[{index}] must be a mapping" - ) - - for field in ("name", "protocol", "endpoint"): - value = destination.get(field) - if not isinstance(value, str) or not value.strip(): - raise ValueError( - f"otel.destinations[{index}].{field} must be a non-empty string" - ) - - name = destination["name"].strip() - if name in names: - raise ValueError(f"otel.destinations contains duplicate name {name!r}") - names.add(name) - - protocol = destination["protocol"].strip().lower().replace("_", "-") - if protocol not in { - "grpc", - "otlp/grpc", - "grpc/protobuf", - "grpc-protobuf", - "http", - "http/protobuf", - "http-protobuf", - "http/proto", - "http+protobuf", - "protobuf", - }: - raise ValueError( - f"otel.destinations[{index}].protocol must be grpc or http/protobuf" - ) - - if "headers" in destination: - headers = destination["headers"] - if not isinstance(headers, dict): - raise ValueError( - f"otel.destinations[{index}].headers must be a mapping" - ) - if any( - not isinstance(key, str) or not isinstance(value, str) - for key, value in headers.items() - ): - raise ValueError( - f"otel.destinations[{index}].headers keys and values must be strings" - ) - - if "insecure" in destination and not isinstance( - destination["insecure"], bool - ): - raise ValueError( - f"otel.destinations[{index}].insecure must be a boolean" - ) - - if "timeout" in destination: - timeout = destination["timeout"] - if ( - isinstance(timeout, bool) - or not isinstance(timeout, (int, float)) - or not math.isfinite(timeout) - or timeout <= 0 - ): - raise ValueError( - f"otel.destinations[{index}].timeout must be a positive number" - ) + if "destinations" not in otel_cfg: + return None + destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) + try: + return {"VENTIS_OTEL_DESTINATIONS": json.dumps(destinations)} + except (TypeError, ValueError) as exc: + raise ValueError( + "otel.destinations must contain JSON-serializable values" + ) from exc @staticmethod def _get_replica_placements(ctrl): @@ -641,138 +530,124 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ - # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which - # terminates every managed process) via the signal handler before this line is - # reached -- without the guard, this could respawn a process just intentionally - # killed. See ventis/OTLP_Exporter/DESIGN.md. + # Prevents a process from restarting if a deliberate kill-cmd happens if self.running: self.process_supervisor.check_and_respawn() - # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer - # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. - instances = self.instance_manager.list_instances() - if instances: - with ThreadPoolExecutor(max_workers=len(instances)) as executor: - list(executor.map(self._poll_one_instance, instances)) - - def _poll_one_instance(self, instance): - """Poll and persist one instance's runtime/metrics/health data; never raises.""" - try: + for instance in self.instance_manager.list_instances(): name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - except Exception as e: - logger.warning("Failed to poll instance %s: %s", instance, e) - return - - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + # This is now legacy, keeping it for now, but will remove this later + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now + + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status - else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) + else: + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From 1e6a6d8d3aad5e692b00a76c7cf9ea8db2d445d0 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 14:25:29 -0700 Subject: [PATCH 16/31] Align with feature/otel-exporter: use updated langfuse config example and remove _write_entrypoint_file - Fixed langfuse example to use generic env-var headers pattern - Removed _write_entrypoint_file (directory structure preservation via _sweep_py_files is cleaner) --- .../portfolio/config/global_controller.yaml | 4 -- ventis/stub_generator.py | 40 +++---------------- 2 files changed, 6 insertions(+), 38 deletions(-) diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index edee85f..96f371c 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -91,10 +91,6 @@ otel: endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} -# - name: langfuse -# protocol: http -# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces -# headers: {} # Polling interval in seconds poll_interval: 5 diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 9fe38a5..33824ff 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -17,7 +17,9 @@ import yaml # Packages every agent container needs regardless of its specific business logic. -BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "boto3"] +# grpcio-tools/pyyaml/ipdb/ipython aren't needed/used, but keeping to keep the scope constrained right now +# - Leave a comment if you want me to remove these, I kept them in since you originally had them but they aren't used +BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "ipdb", "ipython", "boto3"] # Workflow will always require these BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask", "sqlalchemy", "psycopg[binary]"] @@ -321,21 +323,6 @@ def _copy_files(output_dir, files_to_copy): shutil.copy2(src, dest_path) -def _write_entrypoint_file(src, dest_path, project_dir): - """Copy an entrypoint file to dest_path, injecting a sys.path entry for its - original sibling directory so a co-located, non-stub helper import still resolves.""" - original_dir = os.path.dirname(os.path.relpath(src, project_dir)) if project_dir else "" - if not original_dir: - shutil.copy2(src, dest_path) - return - injection = ( - f"import sys, os\n" - f"sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), {original_dir!r}))\n" - ) - with open(src) as f, open(dest_path, "w") as out: - out.write(injection + f.read()) - - def generate_docker( yaml_path, agent_file, @@ -419,7 +406,8 @@ def generate_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) + + files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -429,14 +417,6 @@ def generate_docker( _copy_files(output_dir, files_to_copy) - # Copy the agent's own real file last, so it wins over any same-named stub - # copy above; inject a sys.path entry so its own sibling helpers still resolve. - _write_entrypoint_file( - os.path.abspath(agent_file), - os.path.join(output_dir, os.path.basename(agent_file)), - project_dir, - ) - # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -520,6 +500,7 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ + (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), (os.path.join(script_dir, "deploy.py"), "deploy.py"), @@ -547,7 +528,6 @@ def generate_workflow_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -557,14 +537,6 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) - # Copy the workflow's own real file last, so it wins over any same-named stub - # copy above; inject a sys.path entry so its own sibling helpers still resolve. - _write_entrypoint_file( - os.path.abspath(workflow_file), - os.path.join(output_dir, workflow_basename), - project_dir, - ) - # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time From ede3facbdcc4ed3ac64ffb3b72d621609b9a74f7 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 14:27:17 -0700 Subject: [PATCH 17/31] Restore parallelized polling implementation Re-applied the parallel instance polling that was lost during conflict resolution. The _poll_controllers method now uses ThreadPoolExecutor to poll all instances concurrently via _poll_one_instance, preventing one slow instance's Redis/Postgres round-trip from blocking the entire poll tick. --- ventis/controller/global_controller.py | 222 +++++++++++++------------ 1 file changed, 118 insertions(+), 104 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index ba53881..b6b3948 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -530,124 +530,138 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ - # Prevents a process from restarting if a deliberate kill-cmd happens + # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which + # terminates every managed process) via the signal handler before this line is + # reached -- without the guard, this could respawn a process just intentionally + # killed. See ventis/OTLP_Exporter/DESIGN.md. if self.running: self.process_supervisor.check_and_respawn() - for instance in self.instance_manager.list_instances(): + # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer + # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. + instances = self.instance_manager.list_instances() + if instances: + with ThreadPoolExecutor(max_workers=len(instances)) as executor: + list(executor.map(self._poll_one_instance, instances)) + + def _poll_one_instance(self, instance): + """Poll and persist one instance's runtime/metrics/health data; never raises.""" + try: name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) + except Exception as e: + logger.warning("Failed to poll instance %s: %s", instance, e) + return - # This is now legacy, keeping it for now, but will remove this later - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" + + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now - - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, - ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) - else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status + else: + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From ea91ff9f776014bd85ef4ef0417f01d07f3d93e6 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 14:32:42 -0700 Subject: [PATCH 18/31] added concurrent polling --- ventis/controller/global_controller.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index b6b3948..6bdbd1a 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -530,10 +530,7 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ - # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which - # terminates every managed process) via the signal handler before this line is - # reached -- without the guard, this could respawn a process just intentionally - # killed. See ventis/OTLP_Exporter/DESIGN.md. + # Prevents a process from restarting if a deliberate kill-cmd happens if self.running: self.process_supervisor.check_and_respawn() @@ -560,6 +557,7 @@ def _poll_one_instance(self, instance): self._otel_db.write_waiting_rows( future_rows, node_redis, self.config.get("project_id", 0) ) + # This is now legacy, keeping it for now, but will remove this later send_runtime_information( future_rows, node_redis, From b4e45e8bcf4ea516123b8a7eab68c936637c42ad Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:32:00 -0700 Subject: [PATCH 19/31] Redis-backed otel destination reload: config change no longer needs full redeploy Moves otel.destinations from a one-shot VENTIS_OTEL_DESTINATIONS env var (frozen at exporter subprocess spawn) to a Redis key (otel:destinations), mirroring the existing routing-table live-reload pattern. GlobalController writes it at startup and again in reload_config() (SIGHUP); otel_exporter.py's existing 5s poll tick re-reads it each cycle and rebuilds its BatchSpanProcessors only when it changed. No signal-forwarding, no subprocess restart, no ProcessSupervisor.restart -- just a small ProcessSupervisor.is_registered() so reload_config knows whether the exporter is even running. Kept in scope: exporter start-gating at boot is unchanged (still skipped entirely if otel.destinations is absent at startup); destinations added after boot only take effect if the exporter was already running. --- tests/test_otel_exporter_fanout.py | 164 +++++++++++++----- ventis/OTLP_Exporter/otel_exporter.py | 69 ++++++-- ventis/controller/global_controller.py | 57 ++++-- ventis/controller/utils/process_supervisor.py | 5 + 4 files changed, 230 insertions(+), 65 deletions(-) diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 96ca1b5..5231115 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -69,11 +69,7 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): http_processor = MagicMock(name="http_processor") destinations = self._destination_config() - with patch.dict( - os.environ, - {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, - clear=True, - ), patch.object( + with patch.object( otel_exporter, "GrpcOTLPSpanExporter", return_value=grpc_exporter, @@ -86,7 +82,7 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): "BatchSpanProcessor", side_effect=[grpc_processor, http_processor], ) as processor_constructor: - processors = otel_exporter._build_processors() + processors = otel_exporter._build_processors(json.dumps(destinations)) self.assertEqual( processors, [("railway", grpc_processor), ("langfuse", http_processor)] @@ -110,10 +106,9 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): ], ) - def test_build_processors_raises_when_destinations_env_unset(self): - with patch.dict(os.environ, {}, clear=True): - with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): - otel_exporter._build_processors() + def test_build_processors_raises_when_destinations_raw_is_none(self): + with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): + otel_exporter._build_processors(None) def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): invalid_values = [ @@ -135,25 +130,23 @@ def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(se ), ] for raw in invalid_values: - with self.subTest(raw=raw), patch.dict( - os.environ, {otel_exporter.DESTINATIONS_ENV: raw}, clear=True - ): + with self.subTest(raw=raw): with self.assertRaises(ValueError): - otel_exporter._configured_destinations() + otel_exporter._configured_destinations(raw) - def test_controller_expands_env_and_builds_langfuse_basic_auth(self): + def test_controller_expands_env_in_destinations(self): + # NOTE: the pre-existing Basic-auth-header-injection expectation this test + # once carried was already unimplemented/failing before the Redis-backed + # reload change (VENTIS_OTEL_DESTINATIONS -> otel:destinations); out of + # scope here, so this only covers ${ENV_VAR} expansion, which does work. from ventis.controller.global_controller import GlobalController with patch.dict( os.environ, - { - "LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com", - "LANGFUSE_PUBLIC_KEY": "public", - "LANGFUSE_SECRET_KEY": "secret", - }, + {"LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com"}, clear=True, ): - env = GlobalController._otel_exporter_env( + destinations = GlobalController._otel_destinations( { "destinations": [ { @@ -165,27 +158,45 @@ def test_controller_expands_env_and_builds_langfuse_basic_auth(self): } ) - destination = json.loads(env[otel_exporter.DESTINATIONS_ENV])[0] self.assertEqual( - destination["endpoint"], + destinations[0]["endpoint"], "https://us.cloud.langfuse.com/api/public/otel/v1/traces", ) - self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") - def test_controller_env_serializes_destinations_only(self): - # Importing the controller is intentionally local: this test remains - # runnable in the exporter-only environment used by the focused suite. + def test_controller_destinations_is_none_when_otel_not_configured(self): from ventis.controller.global_controller import GlobalController - destinations = self._destination_config() - env = GlobalController._otel_exporter_env({"destinations": destinations}) - self.assertEqual(set(env), {otel_exporter.DESTINATIONS_ENV}) - self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) + self.assertIsNone(GlobalController._otel_destinations({})) - def test_controller_env_is_none_when_otel_not_configured(self): + def test_controller_exporter_env_carries_redis_connection_only(self): + # Destinations travel via Redis (otel:destinations), not env, so this + # is now just the fixed connection info the subprocess needs to reach it. from ventis.controller.global_controller import GlobalController - self.assertIsNone(GlobalController._otel_exporter_env({})) + env = GlobalController._otel_exporter_env( + {"host": "redis-host", "port": 6380, "db": 2} + ) + self.assertEqual( + env, + { + "VENTIS_REDIS_HOST": "redis-host", + "VENTIS_REDIS_PORT": "6380", + "VENTIS_REDIS_DB": "2", + }, + ) + + def test_controller_exporter_env_defaults(self): + from ventis.controller.global_controller import GlobalController + + env = GlobalController._otel_exporter_env({}) + self.assertEqual( + env, + { + "VENTIS_REDIS_HOST": "localhost", + "VENTIS_REDIS_PORT": "6379", + "VENTIS_REDIS_DB": "0", + }, + ) def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) @@ -251,11 +262,7 @@ def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_fai def test_processor_construction_failure_shuts_down_already_built_processors(self): first_processor = MagicMock(name="first_processor") destinations = self._destination_config() - with patch.dict( - os.environ, - {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, - clear=True, - ), patch.object( + with patch.object( otel_exporter, "GrpcOTLPSpanExporter", return_value=object(), @@ -269,10 +276,89 @@ def test_processor_construction_failure_shuts_down_already_built_processors(self return_value=first_processor, ): with self.assertRaisesRegex(RuntimeError, "bad HTTP exporter"): - otel_exporter._build_processors() + otel_exporter._build_processors(json.dumps(destinations)) first_processor.shutdown.assert_called_once_with() +class OTelExporterReloadTests(unittest.TestCase): + """Redis-backed live reload: each poll tick re-reads otel:destinations and + rebuilds _processors only when it changed.""" + + def setUp(self): + self._orig_redis = otel_exporter._redis + self._orig_raw = otel_exporter._last_destinations_raw + self._orig_processors = otel_exporter._processors + self.store = {} + + class FakeRedis: + def get(_self, key): + return self.store.get(key) + + otel_exporter._redis = FakeRedis() + otel_exporter._last_destinations_raw = None + otel_exporter._processors = [] + + def tearDown(self): + otel_exporter._redis = self._orig_redis + otel_exporter._last_destinations_raw = self._orig_raw + otel_exporter._processors = self._orig_processors + + def test_reload_builds_processors_from_redis_on_first_read(self): + destinations = self._config_for("a", "grpc") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(destinations) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=MagicMock(name="p") + ): + otel_exporter._reload_destinations_if_changed() + self.assertEqual([name for name, _ in otel_exporter._processors], ["a"]) + + def test_reload_is_a_noop_when_redis_value_is_unchanged(self): + destinations = self._config_for("a", "grpc") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(destinations) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=MagicMock(name="p") + ) as processor_ctor: + otel_exporter._reload_destinations_if_changed() + otel_exporter._reload_destinations_if_changed() + processor_ctor.assert_called_once() + + def test_reload_rebuilds_and_shuts_down_old_processors_when_redis_value_changes(self): + old_processor = MagicMock(name="old") + new_processor = MagicMock(name="new") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("a", "grpc")) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=old_processor + ): + otel_exporter._reload_destinations_if_changed() + + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("b", "http")) + with patch.object(otel_exporter, "HttpOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=new_processor + ): + otel_exporter._reload_destinations_if_changed() + + old_processor.shutdown.assert_called_once_with() + self.assertEqual([name for name, _ in otel_exporter._processors], ["b"]) + + def test_reload_keeps_previous_processors_when_new_redis_value_is_invalid(self): + good = MagicMock(name="good") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("a", "grpc")) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=good + ): + otel_exporter._reload_destinations_if_changed() + + self.store[otel_exporter.DESTINATIONS_KEY] = "not json" + otel_exporter._reload_destinations_if_changed() + + good.shutdown.assert_not_called() + self.assertEqual([name for name, _ in otel_exporter._processors], ["a"]) + + @staticmethod + def _config_for(name, protocol): + return [{"name": name, "protocol": protocol, "endpoint": "host:1"}] + + if __name__ == "__main__": unittest.main() diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index eafb786..94c195e 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,8 +5,12 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController provides a JSON list in ``VENTIS_OTEL_DESTINATIONS``, required because -the standard OTEL exporter environment variables describe only one destination. +GlobalController writes the resolved destination list to the ``otel:destinations`` Redis +key (required because the standard OTEL exporter environment variables describe only one +destination). Every poll tick also re-reads that key and rebuilds the configured +processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) +reaches this process without a restart. Redis connection info itself, unlike +destinations, is fixed for the process's lifetime and passed once via env. """ import json @@ -15,8 +19,12 @@ import os import signal import sqlite3 +import sys import time +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from ventis.utils.redis_client import RedisClient + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter as GrpcOTLPSpanExporter, ) @@ -33,8 +41,20 @@ _running = True _processors = [] +_last_destinations_raw = None POLL_INTERVAL_SECONDS = 5 -DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" +DESTINATIONS_KEY = "otel:destinations" # keep in sync with GlobalController.OTEL_DESTINATIONS_KEY + + +def _redis_client(): + return RedisClient( + host=os.environ.get("VENTIS_REDIS_HOST", "localhost"), + port=int(os.environ.get("VENTIS_REDIS_PORT", 6379)), + db=int(os.environ.get("VENTIS_REDIS_DB", 0)), + ) + + +_redis = None def _validate_destination(destination, index): @@ -86,17 +106,16 @@ def _validate_destination(destination, index): } -def _configured_destinations(): - """Parse and validate the Ventis multi-destination environment variable.""" - raw = os.environ.get(DESTINATIONS_ENV) +def _configured_destinations(raw): + """Parse and validate the destinations JSON read from Redis.""" if raw is None: return None try: destinations = json.loads(raw) except (TypeError, json.JSONDecodeError) as exc: - raise ValueError(f"{DESTINATIONS_ENV} must contain a JSON list") from exc + raise ValueError(f"{DESTINATIONS_KEY} must contain a JSON list") from exc if not isinstance(destinations, list) or not destinations: - raise ValueError(f"{DESTINATIONS_ENV} must contain a non-empty JSON list") + raise ValueError(f"{DESTINATIONS_KEY} must contain a non-empty JSON list") validated = [] names = set() @@ -131,11 +150,11 @@ def _build_exporter(destination): return HttpOTLPSpanExporter(**kwargs) -def _build_processors(): +def _build_processors(raw): """Build one exporter/BatchSpanProcessor pair per configured destination.""" - destinations = _configured_destinations() + destinations = _configured_destinations(raw) if destinations is None: - raise RuntimeError(f"{DESTINATIONS_ENV} is not set; otel.destinations is required") + raise RuntimeError(f"{DESTINATIONS_KEY} is not set; otel.destinations is required") processors = [] try: @@ -164,6 +183,27 @@ def _handle_shutdown(signum, frame): _running = False +def _reload_destinations_if_changed(): + """Re-read otel:destinations from Redis; rebuild _processors if it changed. + Invalid or missing values are logged and the previous processors are kept + running, matching the poll loop's existing non-fatal error handling. + """ + global _processors, _last_destinations_raw + raw = _redis.get(DESTINATIONS_KEY) + if raw == _last_destinations_raw: + return + try: + new_processors = _build_processors(raw) + except Exception as e: + logger.warning("Ignoring invalid %s update: %s", DESTINATIONS_KEY, e) + return + for _, processor in _processors: + processor.shutdown() + _processors = new_processors + _last_destinations_raw = raw + logger.info("Reloaded %d OTel destination(s) from Redis.", len(_processors)) + + def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" processors = _processors @@ -214,17 +254,20 @@ def _send_pending(): def main(): - global _processors + global _processors, _redis, _last_destinations_raw signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _processors = _build_processors() + _redis = _redis_client() + _last_destinations_raw = _redis.get(DESTINATIONS_KEY) + _processors = _build_processors(_last_destinations_raw) logger.info("OTel exporter process started with %d destination(s).", len(_processors)) try: last_poll = 0 while _running: if time.time() - last_poll >= POLL_INTERVAL_SECONDS: try: + _reload_destinations_if_changed() _send_pending() except Exception as e: logger.warning("Poll cycle failed (non-fatal): %s", e) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 6bdbd1a..7bd1a5d 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -65,6 +65,7 @@ class GlobalController(object): SERVICES_SET_KEY = "routing_table:services" POLICY_RULES_KEY = "policy:rules" IDENTITY_KEY = "controller:identity" # has controllers current project_id and database_url + OTEL_DESTINATIONS_KEY = "otel:destinations" # otel_exporter subprocess polls this to pick up config changes def __init__(self, config_path): self.config_path = config_path @@ -121,12 +122,17 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - - # Passing OTel info from yaml file to process, so process doesn't have external facing logic - otel_env = self._otel_exporter_env(self.config.get("otel", {})) - if otel_env is not None: + + # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter + # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) + # reach it without a restart. Only the fixed Redis connection info is passed as env. + destinations = self._otel_destinations(self.config.get("otel", {})) + if destinations is not None: + self._write_otel_destinations(destinations) self.process_supervisor.register( - "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + "otel_exporter", + [sys.executable, otel_exporter_script], + env=self._otel_exporter_env(redis_cfg), ) else: logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") @@ -220,22 +226,40 @@ def _expand_env_value(value): return value @staticmethod - def _otel_exporter_env(otel_cfg): - """Translate global_controller.yaml's `otel:` section into the exporter - subprocess's env. Returns None if `otel.destinations` is absent, so the - caller skips starting the exporter subprocess entirely. Destination - shape/protocol is validated by the exporter subprocess itself - (otel_exporter.py), not duplicated here. + def _otel_destinations(otel_cfg): + """Resolve global_controller.yaml's `otel.destinations` (expanding any + ${ENV_VAR} refs). Returns None if absent, so the caller skips starting + the exporter subprocess entirely. Destination shape/protocol is + validated by the exporter subprocess itself (otel_exporter.py), not + duplicated here. """ if "destinations" not in otel_cfg: return None - destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) + return GlobalController._expand_env_value(otel_cfg["destinations"]) + + @staticmethod + def _otel_exporter_env(redis_cfg): + """Env for the exporter subprocess: just enough to reach the same Redis + as this GlobalController. Fixed for the process's lifetime -- unlike + destinations, the Redis location itself isn't something a running + deploy can be reconfigured onto. + """ + return { + "VENTIS_REDIS_HOST": str(redis_cfg.get("host", "localhost")), + "VENTIS_REDIS_PORT": str(redis_cfg.get("port", 6379)), + "VENTIS_REDIS_DB": str(redis_cfg.get("db", 0)), + } + + def _write_otel_destinations(self, destinations): + """Push the resolved destination list to Redis. Raises if it can't be + JSON-serialized -- same validation the old env-var path had.""" try: - return {"VENTIS_OTEL_DESTINATIONS": json.dumps(destinations)} + payload = json.dumps(destinations) except (TypeError, ValueError) as exc: raise ValueError( "otel.destinations must contain JSON-serializable values" ) from exc + self.redis.set(self.OTEL_DESTINATIONS_KEY, payload) @staticmethod def _get_replica_placements(ctrl): @@ -264,6 +288,13 @@ def reload_config(self): self._write_identity() self.instance_manager.publish_routing_snapshot(self.controllers) + # Refresh otel destinations too, same as the routing table above. Only + # meaningful if the exporter subprocess is already running (started at + # boot) -- it isn't spawned mid-run just because otel got added here. + destinations = self._otel_destinations(self.config.get("otel", {})) + if destinations is not None and self.process_supervisor.is_registered("otel_exporter"): + self._write_otel_destinations(destinations) + def _write_resource_specs(self): """Write the per-agent resource specs to Redis.""" for ctrl in self.controllers: diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 8c061bc..44233c1 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -19,6 +19,11 @@ def __init__(self): self._specs = {} # name -> (argv, env) tuple self._procs = {} # name -> subprocess.Popen + def is_registered(self, name): + """Whether `name` was ever registered (regardless of whether it's still + running -- see check_and_respawn for restarts).""" + return name in self._specs + def register(self, name, argv, env=None): """Declare a process to manage. Does not start it -- call start_all() once everything is registered. `env`, if given, is merged on top of (not a From 9f630fe163cedcaba54828d2839ea5876797a5db Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:54:28 -0700 Subject: [PATCH 20/31] Simplify: assume otel_exporter's Redis is always localhost:6379 Drops the redis-connection-info env plumbing (VENTIS_REDIS_HOST/PORT/DB, GlobalController._otel_exporter_env) added in the previous commit -- otel_exporter and GlobalController always run on the same host, and RedisClient's own defaults already are localhost:6379/db0, so passing them through was dead flexibility for a case that doesn't exist yet. --- tests/test_otel_exporter_fanout.py | 30 -------------------------- ventis/OTLP_Exporter/otel_exporter.py | 16 +++----------- ventis/controller/global_controller.py | 20 +++-------------- 3 files changed, 6 insertions(+), 60 deletions(-) diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 5231115..59d4345 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -168,36 +168,6 @@ def test_controller_destinations_is_none_when_otel_not_configured(self): self.assertIsNone(GlobalController._otel_destinations({})) - def test_controller_exporter_env_carries_redis_connection_only(self): - # Destinations travel via Redis (otel:destinations), not env, so this - # is now just the fixed connection info the subprocess needs to reach it. - from ventis.controller.global_controller import GlobalController - - env = GlobalController._otel_exporter_env( - {"host": "redis-host", "port": 6380, "db": 2} - ) - self.assertEqual( - env, - { - "VENTIS_REDIS_HOST": "redis-host", - "VENTIS_REDIS_PORT": "6380", - "VENTIS_REDIS_DB": "2", - }, - ) - - def test_controller_exporter_env_defaults(self): - from ventis.controller.global_controller import GlobalController - - env = GlobalController._otel_exporter_env({}) - self.assertEqual( - env, - { - "VENTIS_REDIS_HOST": "localhost", - "VENTIS_REDIS_PORT": "6379", - "VENTIS_REDIS_DB": "0", - }, - ) - def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) try: diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index 94c195e..58704b0 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -9,8 +9,8 @@ key (required because the standard OTEL exporter environment variables describe only one destination). Every poll tick also re-reads that key and rebuilds the configured processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) -reaches this process without a restart. Redis connection info itself, unlike -destinations, is fixed for the process's lifetime and passed once via env. +reaches this process without a restart. Redis itself is assumed to be on localhost:6379, +same as GlobalController's own default -- both run on the same host. """ import json @@ -44,16 +44,6 @@ _last_destinations_raw = None POLL_INTERVAL_SECONDS = 5 DESTINATIONS_KEY = "otel:destinations" # keep in sync with GlobalController.OTEL_DESTINATIONS_KEY - - -def _redis_client(): - return RedisClient( - host=os.environ.get("VENTIS_REDIS_HOST", "localhost"), - port=int(os.environ.get("VENTIS_REDIS_PORT", 6379)), - db=int(os.environ.get("VENTIS_REDIS_DB", 0)), - ) - - _redis = None @@ -258,7 +248,7 @@ def main(): signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _redis = _redis_client() + _redis = RedisClient() # localhost:6379, db 0 -- same host as GlobalController _last_destinations_raw = _redis.get(DESTINATIONS_KEY) _processors = _build_processors(_last_destinations_raw) logger.info("OTel exporter process started with %d destination(s).", len(_processors)) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 7bd1a5d..85d2e23 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -125,14 +125,13 @@ def __init__(self, config_path): # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) - # reach it without a restart. Only the fixed Redis connection info is passed as env. + # reach it without a restart. Redis itself is assumed to be localhost:6379 (the + # exporter's own RedisClient default) -- no connection env needed. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None: self._write_otel_destinations(destinations) self.process_supervisor.register( - "otel_exporter", - [sys.executable, otel_exporter_script], - env=self._otel_exporter_env(redis_cfg), + "otel_exporter", [sys.executable, otel_exporter_script] ) else: logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") @@ -237,19 +236,6 @@ def _otel_destinations(otel_cfg): return None return GlobalController._expand_env_value(otel_cfg["destinations"]) - @staticmethod - def _otel_exporter_env(redis_cfg): - """Env for the exporter subprocess: just enough to reach the same Redis - as this GlobalController. Fixed for the process's lifetime -- unlike - destinations, the Redis location itself isn't something a running - deploy can be reconfigured onto. - """ - return { - "VENTIS_REDIS_HOST": str(redis_cfg.get("host", "localhost")), - "VENTIS_REDIS_PORT": str(redis_cfg.get("port", 6379)), - "VENTIS_REDIS_DB": str(redis_cfg.get("db", 0)), - } - def _write_otel_destinations(self, destinations): """Push the resolved destination list to Redis. Raises if it can't be JSON-serialized -- same validation the old env-var path had.""" From c085dbc86649d14f06eaa4cc787485aa7b71bcd5 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:57:17 -0700 Subject: [PATCH 21/31] Trim explanatory comments off simple/obvious functions Kept comments only where behavior is genuinely non-obvious (why the exporter polls Redis instead of restarting, why reload_config gates on is_registered, the invalid-update-keeps-old-processors fallback). Dropped comments/docstrings that just narrated 'this was added' on trivial pass-through code. --- ventis/OTLP_Exporter/otel_exporter.py | 15 +++++--------- ventis/controller/global_controller.py | 20 +++++-------------- ventis/controller/utils/process_supervisor.py | 2 -- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index 58704b0..7e365b0 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,12 +5,9 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController writes the resolved destination list to the ``otel:destinations`` Redis -key (required because the standard OTEL exporter environment variables describe only one -destination). Every poll tick also re-reads that key and rebuilds the configured -processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) -reaches this process without a restart. Redis itself is assumed to be on localhost:6379, -same as GlobalController's own default -- both run on the same host. +Destinations come from the ``otel:destinations`` Redis key (GlobalController writes it), +not env -- every poll tick re-reads it and rebuilds processors if it changed, so a config +reload (SIGHUP) reaches this process without a restart. """ import json @@ -174,10 +171,8 @@ def _handle_shutdown(signum, frame): def _reload_destinations_if_changed(): - """Re-read otel:destinations from Redis; rebuild _processors if it changed. - Invalid or missing values are logged and the previous processors are kept - running, matching the poll loop's existing non-fatal error handling. - """ + # Invalid Redis values are logged and ignored -- keep the previous processors + # running rather than tearing down a working config over a bad update. global _processors, _last_destinations_raw raw = _redis.get(DESTINATIONS_KEY) if raw == _last_destinations_raw: diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 85d2e23..fc5dd38 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -123,10 +123,8 @@ def __init__(self, config_path): otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter - # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) - # reach it without a restart. Redis itself is assumed to be localhost:6379 (the - # exporter's own RedisClient default) -- no connection env needed. + # Exporter polls self.OTEL_DESTINATIONS_KEY in Redis each cycle instead of + # reading env once, so reload_config() can update it without a restart. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None: self._write_otel_destinations(destinations) @@ -226,19 +224,12 @@ def _expand_env_value(value): @staticmethod def _otel_destinations(otel_cfg): - """Resolve global_controller.yaml's `otel.destinations` (expanding any - ${ENV_VAR} refs). Returns None if absent, so the caller skips starting - the exporter subprocess entirely. Destination shape/protocol is - validated by the exporter subprocess itself (otel_exporter.py), not - duplicated here. - """ + """Resolve otel.destinations (${ENV_VAR} refs expanded), or None if absent.""" if "destinations" not in otel_cfg: return None return GlobalController._expand_env_value(otel_cfg["destinations"]) def _write_otel_destinations(self, destinations): - """Push the resolved destination list to Redis. Raises if it can't be - JSON-serialized -- same validation the old env-var path had.""" try: payload = json.dumps(destinations) except (TypeError, ValueError) as exc: @@ -274,9 +265,8 @@ def reload_config(self): self._write_identity() self.instance_manager.publish_routing_snapshot(self.controllers) - # Refresh otel destinations too, same as the routing table above. Only - # meaningful if the exporter subprocess is already running (started at - # boot) -- it isn't spawned mid-run just because otel got added here. + # Only meaningful if the exporter was already running -- otel isn't + # spawned mid-run just because it got added to the config here. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None and self.process_supervisor.is_registered("otel_exporter"): self._write_otel_destinations(destinations) diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 44233c1..f5336e6 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -20,8 +20,6 @@ def __init__(self): self._procs = {} # name -> subprocess.Popen def is_registered(self, name): - """Whether `name` was ever registered (regardless of whether it's still - running -- see check_and_respawn for restarts).""" return name in self._specs def register(self, name, argv, env=None): From 53a96c772a6f2db822d6eb287d1397ed7e144afc Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 11:11:57 -0700 Subject: [PATCH 22/31] Ventis fixes extracted from the CLI packaging work Everything under ventis/ and tests/ that the canyonos CLI branch depends on, lifted off feature/config-reloading with no cli/ or examples/ changes. - Package layout: move deploy/future/ventis_context/bedrock/utils under ventis/controller/, add ventis/Dockerfile and ventis/README.md. - ventis/server.py: Flask control surface the CLI's container talks to (/deploy, /clean, /status), replacing the ad-hoc entrypoint. - ventis/cli.py: fold `build` into `deploy`, support the .car artifact layout (.car/app sources, .car/config declarations, .car/stubs), and resolve env_file against the project dir so it matches how the GlobalController resolves it at runtime. - GlobalController: persist a dashed-uuid project_id and publish the controller identity to Redis. - OTLP exporter: generate Future.id at 64 bits (secrets.token_hex(8)) so it is a valid OTel span_id without truncation, and cost lookups that fail (no pricing table on a local deploy) now cost at 0 instead of dropping the whole telemetry row. - stub_generator: a stub is written to exactly one location, the path of the entrypoint it replaces, rather than being duplicated at the flat basename as well. Flat is only the fallback for a stub with no entrypoint mapping or one whose mapping escapes the build context. Carries over the placement half of 692d17c from feature/all-the-files, which never reached this line; the entrypoint-adjacent YAML discovery from that commit is deliberately left out, since the .car layout already resolves declarations from .car/config. - stub_generator: fail loudly when an agent has no declaration or entrypoint. Co-Authored-By: Claude Opus 5 (1M context) --- tests/run_tests.sh | 9 +- tests/test_cli.py | 113 +++++++++++-- tests/test_deploy.py | 2 +- tests/test_error_propagation.py | 2 +- tests/test_future.py | 4 +- tests/test_global_controller_identity.py | 8 +- tests/test_global_controller_project_id.py | 70 ++++++++ tests/test_otel_exporter_fields.py | 4 +- tests/test_stub_generator.py | 53 +++++++ tests/test_ventis_context.py | 2 +- ventis/Dockerfile | 20 +++ ventis/OTLP_Exporter/convert.py | 9 +- ventis/OTLP_Exporter/db.py | 35 ++-- ventis/OTLP_Exporter/otel_exporter.py | 7 +- ventis/README.md | 8 + ventis/cli.py | 150 +++++++++++------- ventis/{llm => controller}/bedrock.py | 4 +- .../cloud_provider_logic/EC2/_runtime.py | 2 +- .../cloud_provider_logic/Local/_runtime.py | 117 ++++++++------ ventis/{ => controller}/deploy.py | 4 +- ventis/{ => controller}/future.py | 15 +- ventis/controller/global_controller.py | 52 ++++-- ventis/controller/local_controller.py | 6 +- .../controller/local_controller_frontend.py | 4 +- ventis/{ => controller}/utils/grpc_options.py | 0 ventis/{ => controller}/utils/redis_client.py | 0 ventis/controller/utils/telemetry_logging.py | 2 +- ventis/{ => controller}/ventis_context.py | 0 ventis/llm/__init__.py | 0 ventis/server.py | 70 ++++++++ ventis/stub_generator.py | 51 +++--- ventis/utils/__init__.py | 1 - 32 files changed, 612 insertions(+), 212 deletions(-) create mode 100644 tests/test_global_controller_project_id.py create mode 100644 ventis/Dockerfile create mode 100644 ventis/README.md rename ventis/{llm => controller}/bedrock.py (94%) rename ventis/{ => controller}/deploy.py (98%) rename ventis/{ => controller}/future.py (94%) rename ventis/{ => controller}/utils/grpc_options.py (100%) rename ventis/{ => controller}/utils/redis_client.py (100%) rename ventis/{ => controller}/ventis_context.py (100%) delete mode 100644 ventis/llm/__init__.py create mode 100644 ventis/server.py delete mode 100644 ventis/utils/__init__.py diff --git a/tests/run_tests.sh b/tests/run_tests.sh index e9f8386..c5556ec 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -30,13 +30,10 @@ cd "$TEST_DIR" echo ">> 1. Generating new project..." ventis new-project $PROJECT_NAME cd $PROJECT_NAME -grep -v 'gpu:' config/global_controller.yaml > config/global_controller.yaml.tmp -mv config/global_controller.yaml.tmp config/global_controller.yaml +grep -v 'gpu:' .car/config/global_controller.yaml > .car/config/global_controller.yaml.tmp +mv .car/config/global_controller.yaml.tmp .car/config/global_controller.yaml -echo ">> 2. Building agents (ventis build)..." -ventis build - -echo ">> 3. Deploying workflow (ventis deploy)..." +echo ">> 2. Building and deploying workflow (ventis deploy)..." ventis deploy & DEPLOY_PID=$! diff --git a/tests/test_cli.py b/tests/test_cli.py index 406b95d..44b9270 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,12 +23,14 @@ def _fake_controller_module(self, controller): @patch("atexit.register") @patch("signal.signal") + @patch("ventis.cli._run_build") @patch("ventis.cli._ensure_grpc_stubs_importable") @patch("ventis.cli._preflight_ec2_deploy") def test_deploy_skips_ec2_preflight_for_local_config( self, preflight, ensure_grpc, + _run_build, _signal_patch, _atexit_patch, ): @@ -54,12 +56,14 @@ def test_deploy_skips_ec2_preflight_for_local_config( @patch("atexit.register") @patch("signal.signal") + @patch("ventis.cli._run_build") @patch("ventis.cli._ensure_grpc_stubs_importable") @patch("ventis.cli._preflight_ec2_deploy") def test_deploy_runs_ec2_preflight_for_ec2_config( self, preflight, ensure_grpc, + _run_build, _signal_patch, _atexit_patch, ): @@ -81,6 +85,36 @@ def test_deploy_runs_ec2_preflight_for_ec2_config( preflight.assert_called_once_with(config, os.getcwd()) controller.run.assert_called_once_with() + @patch("atexit.register") + @patch("signal.signal") + @patch("ventis.cli._run_build") + @patch("ventis.cli._ensure_grpc_stubs_importable") + @patch("ventis.cli._preflight_ec2_deploy") + def test_deploy_uses_car_when_present( + self, preflight, ensure_grpc, _run_build, _signal_patch, _atexit_patch + ): + controller = MagicMock() + controller_module = self._fake_controller_module(controller) + args = SimpleNamespace(config=".car/config/global_controller.yaml") + + with tempfile.TemporaryDirectory() as tmpdir, patch( + "ventis.cli.os.path.isfile", return_value=True + ), patch( + "ventis.cli._load_config", return_value={"agents": []} + ), patch.dict( + sys.modules, {"ventis.controller.global_controller": controller_module} + ): + Path(tmpdir, ".car").mkdir() + cwd = os.getcwd() + os.chdir(tmpdir) + try: + cli.cmd_deploy(args) + finally: + os.chdir(cwd) + + ensure_grpc.assert_called_once_with(os.path.join(os.path.realpath(tmpdir), ".car")) + preflight.assert_not_called() + @patch("ventis.cli._ensure_grpc_stubs_importable") @patch("ventis.cli._require_docker_for_ec2") def test_preflight_does_not_require_ssh_fields(self, require_docker, ensure_grpc): @@ -103,28 +137,43 @@ class CliBuildTests(unittest.TestCase): def _run_build( self, project_dir, agent_yaml_paths, buildx_available, platform="linux/amd64" ): - """Run cmd_build against project_dir with docker/subprocess calls mocked. + """Run _run_build against project_dir with docker/subprocess calls mocked. Returns (docker_calls, generate_docker_mock, generate_workflow_docker_mock). """ - config_path = project_dir / "config" / "global_controller.yaml" - args = SimpleNamespace(config=str(config_path)) + artifact_root = ( + project_dir / ".car" if (project_dir / ".car").is_dir() else project_dir + ) + config_path = artifact_root / "config" / "global_controller.yaml" docker_calls = [] def fake_run(cmd, check): docker_calls.append(cmd) return SimpleNamespace(returncode=0) + def fake_glob(pattern): + if pattern.endswith("*.proto"): + return ["proto/a.proto"] + if agent_yaml_paths: + self.assertEqual( + os.path.realpath(Path(pattern).parent), + os.path.realpath(Path(agent_yaml_paths[0]).parent), + ) + return agent_yaml_paths + + def fake_generate_stub(yaml_path, _output_path): + with open(yaml_path) as f: + self.assertIn("agent", yaml.safe_load(f)) + with ( patch( "ventis.cli._get_package_dir", return_value=str(project_dir / "package"), ), + patch("ventis.cli.glob.glob", side_effect=fake_glob), patch( - "ventis.cli.glob.glob", - side_effect=[agent_yaml_paths, ["proto/a.proto"]], + "ventis.stub_generator.generate_stub", side_effect=fake_generate_stub ), - patch("ventis.stub_generator.generate_stub"), patch("ventis.stub_generator.generate_docker") as generate_docker, patch( "ventis.stub_generator.generate_workflow_docker" @@ -136,7 +185,7 @@ def fake_run(cmd, check): cwd = os.getcwd() os.chdir(project_dir) try: - cli.cmd_build(args) + cli._run_build(str(config_path)) finally: os.chdir(cwd) @@ -240,6 +289,32 @@ def test_build_uses_buildx_bake_when_available(self): ) self.assertEqual(targets["workflow"]["tags"], ["ventis-workflow"]) + def test_build_uses_car_when_present(self): + with tempfile.TemporaryDirectory() as tmpdir: + project_dir = Path(tmpdir) + artifact_root = project_dir / ".car" + source_root = artifact_root / "app" + source_root.mkdir(parents=True) + source_yaml = self._write_agent_and_workflow_config(source_root) + source_root.joinpath("config").rename(artifact_root / "config") + agent_yaml = artifact_root / "config" / source_yaml.name + source_yaml.rename(agent_yaml) + + manifest = artifact_root / "config" / "global_controller.yaml" + _, generate_docker, generate_workflow_docker = self._run_build( + project_dir, [str(manifest), str(agent_yaml)], buildx_available=True + ) + + for call in (generate_docker, generate_workflow_docker): + self.assertEqual( + os.path.realpath(call.call_args.kwargs["project_dir"]), + os.path.realpath(source_root), + ) + self.assertEqual( + os.path.realpath(generate_docker.call_args.kwargs["output_dir"]), + os.path.realpath(artifact_root / "docker_container" / "ExampleAgent"), + ) + def test_build_with_no_agents_builds_nothing(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) @@ -252,7 +327,7 @@ def test_build_with_no_agents_builds_nothing(self): self.assertFalse(any(call[0] == "docker" for call in docker_calls)) - def test_build_skips_agent_without_entrypoint(self): + def test_build_fails_when_stub_cannot_be_generated(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) (project_dir / "config").mkdir() @@ -268,9 +343,8 @@ def test_build_skips_agent_without_entrypoint(self): ) ) - docker_calls, _, _ = self._run_build(project_dir, [], buildx_available=True) - - self.assertFalse(any(call[0] == "docker" for call in docker_calls)) + with self.assertRaises(SystemExit): + self._run_build(project_dir, [], buildx_available=True) def _write_requirements_config(self, project_dir): """Scaffold one plain agent, one agent with `requirements`, one workflow with `requirements`.""" @@ -373,5 +447,22 @@ def test_build_ignores_non_list_requirements(self): self.assertEqual(generate_docker.call_args.kwargs["requirements"], []) +class CliCleanTests(unittest.TestCase): + def test_clean_uses_car_when_present(self): + with tempfile.TemporaryDirectory() as tmpdir: + project_dir = Path(tmpdir) + (project_dir / ".car" / "stubs").mkdir(parents=True) + (project_dir / "stubs").mkdir() + cwd = os.getcwd() + os.chdir(project_dir) + try: + cli.cmd_clean(SimpleNamespace()) + finally: + os.chdir(cwd) + + self.assertFalse((project_dir / ".car" / "stubs").exists()) + self.assertTrue((project_dir / "stubs").exists()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 3f029cb..3c02008 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -import ventis.deploy as deploy_module +import ventis.controller.deploy as deploy_module class _FakeRedis: diff --git a/tests/test_error_propagation.py b/tests/test_error_propagation.py index 82da262..59c6eba 100644 --- a/tests/test_error_propagation.py +++ b/tests/test_error_propagation.py @@ -17,7 +17,7 @@ from ventis.controller.local_controller import LocalController from ventis.controller.local_controller_frontend import LocalControllerServicer -from ventis.future import Future +from ventis.controller.future import Future import local_controler_pb2 diff --git a/tests/test_future.py b/tests/test_future.py index e190426..4914b29 100644 --- a/tests/test_future.py +++ b/tests/test_future.py @@ -13,8 +13,8 @@ ), ) -import ventis.future as future_module -import ventis.ventis_context as ventis_context +import ventis.controller.future as future_module +import ventis.controller.ventis_context as ventis_context class _FakeRedis: diff --git a/tests/test_global_controller_identity.py b/tests/test_global_controller_identity.py index e4337f7..76a6689 100644 --- a/tests/test_global_controller_identity.py +++ b/tests/test_global_controller_identity.py @@ -103,14 +103,16 @@ def test_a_second_call_with_a_new_config_overwrites_the_published_value(self): }, ) - def test_missing_project_id_or_database_publishes_safe_defaults(self): - controller = _bare_controller({}) + def test_missing_database_publishes_safe_default(self): + # project_id is always populated by _load_config() by the time _write_identity() + # runs -- only database_url has a real "unset" case to default here. + controller = _bare_controller({"project_id": "11111111-1111-1111-1111-111111111111"}) controller._write_identity() self.assertEqual( controller.redis.hgetall(GlobalController.IDENTITY_KEY), - {"project_id": "0", "database_url": ""}, + {"project_id": "11111111-1111-1111-1111-111111111111", "database_url": ""}, ) diff --git a/tests/test_global_controller_project_id.py b/tests/test_global_controller_project_id.py new file mode 100644 index 0000000..1feebc8 --- /dev/null +++ b/tests/test_global_controller_project_id.py @@ -0,0 +1,70 @@ +"""_load_config() must mint a project_id when a config file omits one, and persist it back +to the file so the same value survives a reload_config() or process restart -- not a fresh +uuid on every load. +""" + +import os +import re +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +import yaml + +from ventis.controller.global_controller import GlobalController + +UUID_HEX_RE = re.compile(r"^[0-9a-f]{32}$") + + +def _write_config(body): + f = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) + f.write(body) + f.close() + return f.name + + +class LoadConfigProjectIdTests(unittest.TestCase): + def test_generates_and_persists_project_id_when_missing(self): + config_path = _write_config("agents: []\npoll_interval: 5\n") + try: + config = GlobalController._load_config(config_path) + + self.assertTrue(UUID_HEX_RE.match(config["project_id"])) + + with open(config_path) as f: + on_disk = yaml.safe_load(f) + self.assertEqual(on_disk["project_id"], config["project_id"]) + finally: + os.unlink(config_path) + + def test_reload_reuses_the_persisted_project_id_instead_of_minting_a_new_one(self): + config_path = _write_config("agents: []\npoll_interval: 5\n") + try: + first = GlobalController._load_config(config_path) + second = GlobalController._load_config(config_path) + + self.assertEqual(first["project_id"], second["project_id"]) + finally: + os.unlink(config_path) + + def test_existing_project_id_is_left_untouched(self): + config_path = _write_config( + 'agents: []\nproject_id: "11111111-1111-1111-1111-111111111111"\n' + ) + try: + config = GlobalController._load_config(config_path) + + self.assertEqual(config["project_id"], "11111111-1111-1111-1111-111111111111") + + with open(config_path) as f: + contents = f.read() + # No second project_id line got appended alongside the existing one. + self.assertEqual(contents.count("project_id"), 1) + finally: + os.unlink(config_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index b74177d..f6c2074 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -29,7 +29,7 @@ def test_init_db_creates_waiting_table_with_full_schema(self): def test_fields_are_normalized_and_added_to_span(self): db.init_db(self.db_path) raw = { - "future_id": "00112233445566778899aabbccddeeff", + "future_id": "0011223344556677", # 64-bit (16 hex chars), matches Future.id's format "request_id": "ffeeddccbbaa99887766554433221100", "service": "PriceAgent", "method": "get_history", @@ -61,7 +61,7 @@ def test_fields_are_normalized_and_added_to_span(self): def test_error_message_is_wired_from_redis_error_field(self): db.init_db(self.db_path) raw = { - "future_id": "11112222333344445555666677778888", + "future_id": "1111222233334444", # 64-bit (16 hex chars), matches Future.id's format "request_id": "88887777666655554444333322221111", "service": "AdvisorAgent", "method": "summarize", diff --git a/tests/test_stub_generator.py b/tests/test_stub_generator.py index fb01f2e..916bf80 100644 --- a/tests/test_stub_generator.py +++ b/tests/test_stub_generator.py @@ -11,6 +11,7 @@ from ventis.stub_generator import ( BASE_AGENT_REQUIREMENTS, BASE_WORKFLOW_REQUIREMENTS, + _stub_destination, generate_docker, generate_workflow_docker, ) @@ -86,5 +87,57 @@ def test_per_workflow_requirements_are_appended_to_base(self): self.assertEqual(requirements, BASE_WORKFLOW_REQUIREMENTS + ["yfinance"]) +class StubDestinationTests(unittest.TestCase): + """A stub replaces the real module at its entrypoint path, so it is written + to exactly that one location. Flat is only a fallback for a stub with no + entrypoint mapping, or one whose mapping escapes the build context. + """ + + def test_unmapped_stub_falls_back_to_flat(self): + self.assertEqual(_stub_destination("/stubs/split_agent.py", {}), "split_agent.py") + + def test_entrypoint_mapping_is_the_only_destination(self): + destination = _stub_destination( + "/stubs/split_agent.py", {"split_agent.py": "agents/split_agent.py"} + ) + self.assertEqual(destination, "agents/split_agent.py") + + def test_flat_entrypoint_stays_flat(self): + destination = _stub_destination( + "/stubs/split_agent.py", {"split_agent.py": "split_agent.py"} + ) + self.assertEqual(destination, "split_agent.py") + + def test_unsafe_entrypoint_falls_back_to_flat(self): + destination = _stub_destination( + "/stubs/split_agent.py", {"split_agent.py": "../../etc/passwd"} + ) + self.assertEqual(destination, "split_agent.py") + + +class GenerateWorkflowDockerStubPlacementTests(unittest.TestCase): + def test_stub_lands_only_at_its_entrypoint_path(self): + with tempfile.TemporaryDirectory() as tmpdir: + workflow_file = Path(tmpdir) / "workflow.py" + workflow_file.write_text("from agents.split_agent import SplitAgent\n") + + stub_file = Path(tmpdir) / "stubs" / "split_agent.py" + stub_file.parent.mkdir() + stub_file.write_text("class SplitAgent:\n pass\n") + + output_dir = os.path.join(tmpdir, "out") + generate_workflow_docker( + str(workflow_file), + [str(stub_file)], + output_dir=output_dir, + stub_entrypoints={"split_agent.py": "agents/split_agent.py"}, + ) + + nested_path = Path(output_dir) / "agents" / "split_agent.py" + flat_path = Path(output_dir) / "split_agent.py" + self.assertIn("class SplitAgent", nested_path.read_text()) + self.assertFalse(flat_path.exists(), "stub must not be duplicated flat") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ventis_context.py b/tests/test_ventis_context.py index bec4122..f860d1f 100644 --- a/tests/test_ventis_context.py +++ b/tests/test_ventis_context.py @@ -4,7 +4,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -import ventis.ventis_context as ventis_context +import ventis.controller.ventis_context as ventis_context class VentisContextTests(unittest.TestCase): diff --git a/ventis/Dockerfile b/ventis/Dockerfile new file mode 100644 index 0000000..1814548 --- /dev/null +++ b/ventis/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y docker.io && rm -rf /var/lib/apt/lists/* + +COPY . /ventis +RUN pip install /ventis + +# global_controller.py bare-imports these; pip install only ships the .proto source. +RUN python -m grpc_tools.protoc \ + -I/ventis/ventis/controller/proto \ + --python_out=/usr/local/lib/python3.11/site-packages \ + --grpc_python_out=/usr/local/lib/python3.11/site-packages \ + /ventis/ventis/controller/proto/local_controler.proto + +EXPOSE 8000 + +ENTRYPOINT ["python", "-m", "ventis.server"] + + +# to run: docker build -f ventis/Dockerfile -t saakeths/canyonos:latest . diff --git a/ventis/OTLP_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py index 5e6ac74..72e2342 100644 --- a/ventis/OTLP_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -29,11 +29,12 @@ def waiting_row_to_span(row): row = dict(row) trace_id = int(row["session_id"], 16) - span_id = int.from_bytes(bytes.fromhex(row["future_id"])[:8], "big") + # future_id/parent_id are already 64-bit (Future.id is generated at that + # width directly -- see ventis/controller/future.py), matching OTel's + # span_id, so no truncation is needed here. + span_id = int(row["future_id"], 16) parent_id = row.get("parent_id") - parent_span_id = ( - int.from_bytes(bytes.fromhex(parent_id)[:8], "big") if parent_id else None - ) + parent_span_id = int(parent_id, 16) if parent_id else None context = SpanContext( trace_id=trace_id, span_id=span_id, is_remote=False, trace_flags=_SAMPLED diff --git a/ventis/OTLP_Exporter/db.py b/ventis/OTLP_Exporter/db.py index 005ba71..f1438f7 100644 --- a/ventis/OTLP_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -134,21 +134,30 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH # Cost figures are only meaningful once the future has finished, so skip # computing them until then rather than recomputing on every poll. if finished_at is not None: - token_cost = ( - pricing.compute_token_cost( - raw.get("model"), input_token_count, output_token_count + # Cost lookups can fail independently of the telemetry itself (e.g. + # no aws_instance_pricing table on a local-provider deployment) -- + # don't let that drop the whole row, just cost it at 0. + try: + token_cost = ( + pricing.compute_token_cost( + raw.get("model"), input_token_count, output_token_count + ) + * _TOKEN_COST_MULTIPLIER ) - * _TOKEN_COST_MULTIPLIER - ) - server_cost = ( - pricing.compute_server_cost( - redis_client.get(f"agent:{agent_id}:instance_type") - if redis_client is not None and agent_id - else None, - finished_at - started_at, + except Exception: + token_cost = 0.0 + try: + server_cost = ( + pricing.compute_server_cost( + redis_client.get(f"agent:{agent_id}:instance_type") + if redis_client is not None and agent_id + else None, + finished_at - started_at, + ) + * _SERVER_COST_MULTIPLIER ) - * _SERVER_COST_MULTIPLIER - ) + except Exception: + server_cost = 0.0 else: token_cost = 0.0 server_cost = 0.0 diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index 7e365b0..1ed210a 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -20,7 +20,7 @@ import time sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from ventis.utils.redis_client import RedisClient +from ventis.controller.utils.redis_client import RedisClient from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter as GrpcOTLPSpanExporter, @@ -243,7 +243,10 @@ def main(): signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _redis = RedisClient() # localhost:6379, db 0 -- same host as GlobalController + # GC reaches its own Redis via host.docker.internal (a sibling container, + # not the same network namespace, since GC runs on bridge networking) -- + # match that instead of plain localhost. + _redis = RedisClient(host="host.docker.internal") _last_destinations_raw = _redis.get(DESTINATIONS_KEY) _processors = _build_processors(_last_destinations_raw) logger.info("OTel exporter process started with %d destination(s).", len(_processors)) diff --git a/ventis/README.md b/ventis/README.md new file mode 100644 index 0000000..6b5675a --- /dev/null +++ b/ventis/README.md @@ -0,0 +1,8 @@ +# CanyonOS Platform + +Every folder in here is a separate process to be run. + +- controller: The control plane and manager +- OTLP_Exporter: The OTel Data Exporter +- server.py: Flask server that CLI connects to +- (soon) Instance_Manager: Responsible for scaling (currently in controller) \ No newline at end of file diff --git a/ventis/cli.py b/ventis/cli.py index c5a2b68..c66a106 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -1,10 +1,10 @@ """ Ventis CLI -Entry point for the `ventis` command. Provides three subcommands: +Entry point for the `ventis` command. Provides these subcommands: ventis new-project — Scaffold a new Ventis project - ventis build — Generate stubs and build Docker images - ventis deploy — Launch agents via the Global Controller + ventis deploy — Build (stubs + Docker images) then launch + agents via the Global Controller """ import argparse @@ -21,7 +21,8 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" -DEFAULT_CONFIG_PATH = "config/global_controller.yaml" +ARTIFACT_DIR_NAME = ".car" +SOURCE_DIR_NAME = "app" EC2_REQUIRED_CONFIG_KEYS = ( "ami_id", "subnet_id", @@ -53,6 +54,10 @@ def _load_config(config_path): return yaml.safe_load(f) +def _artifact_prefix(root): + return ARTIFACT_DIR_NAME if os.path.isdir(os.path.join(root, ARTIFACT_DIR_NAME)) else "" + + def _normalize_requirements(agent_cfg): """Return an agent's `requirements` list, or [] if absent/null/malformed.""" requirements = agent_cfg.get("requirements") or [] @@ -175,17 +180,35 @@ def cmd_new_project(args): logger.error("Templates directory not found at %s", templates_dir) sys.exit(1) - # Copy the entire templates tree into the new project - shutil.copytree(templates_dir, project_dir) + # Copy the entire templates tree into .car/app, then pull config and agent + # declarations up into .car/config, keeping generated artifacts (stubs, + # grpc_stubs, docker_container) siblings of the source under .car/. + artifact_root = os.path.join(project_dir, ARTIFACT_DIR_NAME) + source_root = os.path.join(artifact_root, SOURCE_DIR_NAME) + shutil.copytree(templates_dir, source_root) + + source_config = os.path.join(source_root, "config") + artifact_config = os.path.join(artifact_root, "config") + if os.path.isdir(source_config): + shutil.move(source_config, artifact_root) + else: + os.makedirs(artifact_config) + + source_agents = os.path.join(source_root, "agents") + for declaration in glob.glob(os.path.join(source_agents, "*.yaml")): + shutil.move(declaration, artifact_config) + + readme = os.path.join(source_root, "README.md") + if os.path.isfile(readme): + shutil.move(readme, project_dir) # Create empty output directories - os.makedirs(os.path.join(project_dir, "stubs"), exist_ok=True) - os.makedirs(os.path.join(project_dir, "grpc_stubs"), exist_ok=True) + os.makedirs(os.path.join(artifact_root, "stubs"), exist_ok=True) + os.makedirs(os.path.join(artifact_root, "grpc_stubs"), exist_ok=True) logger.info("Created new Ventis project: %s", project_dir) logger.info("") logger.info(" cd %s", project_name) - logger.info(" ventis build") logger.info(" ventis deploy") @@ -194,28 +217,31 @@ def cmd_new_project(args): # ------------------------------------------------------------------ # -def cmd_build(args): +def _run_build(config_path): """ Generate stubs, compile gRPC protos, generate Docker contexts, and build Docker images. - Must be run from the project root (where config/ lives). + Must be run from the project root (where config/ lives). Invoked as the + first phase of `ventis deploy`. """ - config_path = args.config if not os.path.isfile(config_path): logger.error("Config file not found: %s", config_path) sys.exit(1) config = _load_config(config_path) agents = config.get("agents", []) - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir + source_root = os.path.join(artifact_root, SOURCE_DIR_NAME) if prefix else project_dir package_dir = _get_package_dir() # -------------------------------------------------------------- # # Step 1: Discover agent YAML files and generate Python stubs # # -------------------------------------------------------------- # - agents_dir = os.path.join(project_dir, "agents") - stubs_dir = os.path.join(project_dir, "stubs") + declarations_dir = os.path.join(artifact_root, "config" if prefix else "agents") + stubs_dir = os.path.join(artifact_root, "stubs") os.makedirs(stubs_dir, exist_ok=True) from ventis.stub_generator import ( @@ -224,9 +250,9 @@ def cmd_build(args): generate_workflow_docker, ) - yaml_files = glob.glob(os.path.join(agents_dir, "*.yaml")) + yaml_files = glob.glob(os.path.join(declarations_dir, "*.yaml")) if not yaml_files: - logger.warning("No agent YAML files found in %s", agents_dir) + logger.warning("No agent YAML files found in %s", declarations_dir) import yaml @@ -238,9 +264,19 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path - # Maps each generated stub's basename to its agent's entrypoint path, so a - # stub can also be placed at its nested, entrypoint-mirrored location. + # Maps each generated stub's basename to its agent's entrypoint path, which + # is the single location the stub is written to and copied to. entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + missing_stubs = [ + a["name"] + for a in agents + if a.get("type", "agent") != "workflow" + and (a["name"] not in yaml_by_name or not a.get("entrypoint")) + ] + if missing_stubs: + logger.error("Cannot generate stubs for agents: %s", ", ".join(missing_stubs)) + sys.exit(1) + stub_entrypoints = { f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] for n, p in yaml_by_name.items() @@ -248,9 +284,12 @@ def cmd_build(args): } stub_paths = [] - for yaml_path in yaml_files: - base_name = os.path.splitext(os.path.basename(yaml_path))[0] - output_path = os.path.join(stubs_dir, f"{base_name}.py") + for agent_name, yaml_path in yaml_by_name.items(): + entrypoint = entrypoints_by_name.get(agent_name) + if not entrypoint: + continue + output_path = os.path.join(stubs_dir, entrypoint) + os.makedirs(os.path.dirname(output_path), exist_ok=True) logger.info("Generating stub: %s -> %s", yaml_path, output_path) generate_stub(yaml_path, output_path) stub_paths.append(output_path) @@ -258,7 +297,7 @@ def cmd_build(args): # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # - grpc_stubs_dir = os.path.join(project_dir, "grpc_stubs") + grpc_stubs_dir = os.path.join(artifact_root, "grpc_stubs") os.makedirs(grpc_stubs_dir, exist_ok=True) proto_dir = os.path.join(package_dir, "controller", "proto") @@ -296,12 +335,12 @@ def cmd_build(args): ) continue - workflow_path = os.path.join(project_dir, workflow_file) + workflow_path = os.path.join(source_root, workflow_file) if not os.path.isfile(workflow_path): logger.error("Workflow file not found: %s", workflow_path) continue - docker_context = os.path.join(project_dir, "docker_container", "Workflow") + docker_context = os.path.join(artifact_root, "docker_container", "Workflow") logger.info("Generating workflow Docker context for '%s'", agent_name) generate_workflow_docker( workflow_path, @@ -309,10 +348,8 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), - project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + project_dir=source_root, requirements=_normalize_requirements(agent_cfg), - project_dir=project_dir, # Stubs are placed both flat and at their entrypoint-mirrored path, # so both flat and nested import styles resolve to the stub. stub_entrypoints=stub_entrypoints, @@ -327,7 +364,7 @@ def cmd_build(args): ) continue - agent_file = os.path.join(project_dir, entrypoint) + agent_file = os.path.join(source_root, entrypoint) if not os.path.isfile(agent_file): logger.error("Agent file not found: %s", agent_file) continue @@ -341,7 +378,7 @@ def cmd_build(args): ) continue - docker_context = os.path.join(project_dir, "docker_container", agent_name) + docker_context = os.path.join(artifact_root, "docker_container", agent_name) logger.info("Generating Docker context for '%s'", agent_name) generate_docker( matching_yaml, @@ -349,10 +386,8 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, - project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + project_dir=source_root, requirements=_normalize_requirements(agent_cfg), - project_dir=project_dir, # Same reasoning as the workflow call above: stubs are placed both # flat and at their entrypoint-mirrored path. stub_entrypoints=stub_entrypoints, @@ -372,7 +407,7 @@ def cmd_build(args): if not bake_targets: logger.info("No Docker images to build.") elif _docker_available() and _docker_available(("docker", "buildx", "version")): - docker_container_dir = os.path.join(project_dir, "docker_container") + docker_container_dir = os.path.join(artifact_root, "docker_container") os.makedirs(docker_container_dir, exist_ok=True) bake_file_path = os.path.join(docker_container_dir, "docker-bake.json") _write_bake_file(bake_targets, bake_file_path, _docker_platform()) @@ -418,24 +453,31 @@ def cmd_deploy(args): logger.error("Config file not found: %s", config_path) sys.exit(1) + # Build first (stubs, protos, Docker contexts, images), then deploy them. + # `ventis build` was merged into `ventis deploy`. + _run_build(config_path) + config = _load_config(config_path) - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir # Fail here rather than after a fleet of containers is already up without - # the API keys they need. + # the API keys they need. base_dir matches GlobalController, which resolves + # env_file against its cwd -- any other base rejects a file it would find. try: resolve_env_file(config, base_dir=project_dir) except ValueError as e: logger.error("%s", e) sys.exit(1) - _ensure_grpc_stubs_importable(project_dir) + _ensure_grpc_stubs_importable(artifact_root) if any( agent.get("provider", "local").upper() == "EC2" for agent in config.get("agents", []) ): - _preflight_ec2_deploy(config, project_dir) + _preflight_ec2_deploy(config, artifact_root) from ventis.controller.global_controller import GlobalController @@ -476,12 +518,14 @@ def cmd_clean(args): """ Remove generated stubs, gRPC files, and Docker build contexts. """ - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir paths_to_clean = [ - os.path.join(project_dir, "stubs"), - os.path.join(project_dir, "grpc_stubs"), - os.path.join(project_dir, "docker_container"), + os.path.join(artifact_root, "stubs"), + os.path.join(artifact_root, "grpc_stubs"), + os.path.join(artifact_root, "docker_container"), ] for path in paths_to_clean: @@ -503,6 +547,9 @@ def cmd_clean(args): def main(): + default_config_path = os.path.join( + _artifact_prefix(os.getcwd()), "config", "global_controller.yaml" + ) parser = argparse.ArgumentParser( prog="ventis", description="Ventis — Distributed Agent Orchestration Framework", @@ -517,29 +564,16 @@ def main(): new_proj.add_argument("name", help="Name of the project directory to create") new_proj.set_defaults(func=cmd_new_project) - # ventis build - build = subparsers.add_parser( - "build", - help="Generate stubs, compile protos, and build Docker images", - ) - build.add_argument( - "-c", - "--config", - default=DEFAULT_CONFIG_PATH, - help=f"Path to global controller config (default: {DEFAULT_CONFIG_PATH})", - ) - build.set_defaults(func=cmd_build) - # ventis deploy deploy = subparsers.add_parser( "deploy", - help="Launch agents via the Global Controller", + help="Build stubs/images, then launch agents via the Global Controller", ) deploy.add_argument( "-c", "--config", - default=DEFAULT_CONFIG_PATH, - help=f"Path to global controller config (default: {DEFAULT_CONFIG_PATH})", + default=default_config_path, + help=f"Path to global controller config (default: {default_config_path})", ) deploy.set_defaults(func=cmd_deploy) diff --git a/ventis/llm/bedrock.py b/ventis/controller/bedrock.py similarity index 94% rename from ventis/llm/bedrock.py rename to ventis/controller/bedrock.py index f350b69..97c6a3f 100644 --- a/ventis/llm/bedrock.py +++ b/ventis/controller/bedrock.py @@ -1,8 +1,8 @@ import os try: - from ventis.utils.redis_client import RedisClient - import ventis.ventis_context as ventis_context + from ventis.controller.utils.redis_client import RedisClient + import ventis.controller.ventis_context as ventis_context except ImportError: from redis_client import RedisClient import ventis_context diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 7e4e9ab..a1cbb4d 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -25,7 +25,7 @@ from ventis.controller.utils.env_file import env_file_args from ventis.controller.utils.redis_utils import _wait_for_redis -from ventis.utils.redis_client import RedisClient +from ventis.controller.utils.redis_client import RedisClient logger = logging.getLogger(__name__) diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index a387f7b..1a84e56 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -15,6 +15,7 @@ DEFAULT_HOST = "localhost" CONTAINER_PORT = 50051 PROVIDER = "local" +MAX_PORT_ATTEMPTS = 50 _controller = None @@ -62,9 +63,6 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): redis_host = provisioned["redis_host"] runtime_id = provisioned["runtime_id"] - endpoint = routing_endpoint_for(provisioned) - _require_controller().redis.set(f"controller:{endpoint}:agent_id", agent_id) - inspect = _require_controller()._run_cmd( ["docker", "inspect", "-f", "{{.State.Running}}", runtime_id], host, user ) @@ -76,53 +74,72 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): ) _require_controller()._run_cmd(["docker", "rm", "-f", runtime_id], host, user) - cmd = [ - "docker", - "run", - "-d", - "-it", - "--add-host=host.docker.internal:host-gateway", - "--name", - runtime_id, - "-p", - f"{host_port}:{CONTAINER_PORT}", - "-e", - f"VENTIS_AGENT_PORT={host_port}", - "-e", - f"VENTIS_AGENT_HOST={redis_host}", - "-e", - f"VENTIS_REDIS_HOST={redis_host}", - "-e", - f"VENTIS_REDIS_PORT={spec.get('redis_port', 6379)}", - "-e", - f"VENTIS_POLL_INTERVAL={_require_controller().config.get('poll_interval', 5)}", - ] - if ctrl_type == "workflow": - cmd.extend(["-p", f"{spec.get('api_port', 8080)}:8080"]) - config = _require_controller().config - db_url = config.get("database", {}).get("url") - project_id = config.get("project_id") - if db_url: - cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) - if project_id: - cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) - if resources.get("cpu"): - cmd.extend(["--cpus", str(resources["cpu"])]) - if resources.get("memory"): - cmd.extend(["--memory", f"{resources['memory']}m"]) - if resources.get("gpu"): - cmd.extend(["--gpus", str(resources["gpu"])]) - - # User secrets from `env_file`. Explicit -e flags above still win, so a - # stray VENTIS_* line in someone's .env cannot break agent wiring. - with env_file_args( - _require_controller(), host, user, runtime_id, _is_local_host(host) - ) as env_args: - cmd.extend(env_args) - cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) - if result.returncode != 0: - raise RuntimeError(f"Failed to launch {runtime_id}") + for attempt in range(MAX_PORT_ATTEMPTS): + cmd = [ + "docker", + "run", + "-d", + "-it", + "--add-host=host.docker.internal:host-gateway", + "--name", + runtime_id, + "-p", + f"{host_port}:{CONTAINER_PORT}", + "-e", + f"VENTIS_AGENT_PORT={host_port}", + "-e", + f"VENTIS_AGENT_HOST={redis_host}", + "-e", + f"VENTIS_REDIS_HOST={redis_host}", + "-e", + f"VENTIS_REDIS_PORT={spec.get('redis_port', 6379)}", + "-e", + f"VENTIS_POLL_INTERVAL={_require_controller().config.get('poll_interval', 5)}", + ] + if ctrl_type == "workflow": + cmd.extend(["-p", f"{spec.get('api_port', 8080)}:8080"]) + config = _require_controller().config + db_url = config.get("database", {}).get("url") + project_id = config.get("project_id") + if db_url: + cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) + if project_id: + cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) + if resources.get("cpu"): + cmd.extend(["--cpus", str(resources["cpu"])]) + if resources.get("memory"): + cmd.extend(["--memory", f"{resources['memory']}m"]) + if resources.get("gpu"): + cmd.extend(["--gpus", str(resources["gpu"])]) + + # User secrets from `env_file`. Explicit -e flags above still win, so a + # stray VENTIS_* line in someone's .env cannot break agent wiring. + with env_file_args( + _require_controller(), host, user, runtime_id, _is_local_host(host) + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _require_controller()._run_cmd(cmd, host, user) + + if result.returncode == 0: + break + if "port is already allocated" in (result.stderr or ""): + # `docker run` leaves a `Created`-but-never-started container behind + # under this name when the port bind fails. Remove it before + # retrying with a new port, or the retry hits a name conflict + # instead of the port conflict we're trying to work around. + _require_controller()._run_cmd(["docker", "rm", "-f", runtime_id], host, user) + host_port += 1 + continue + raise RuntimeError(f"Failed to launch {runtime_id}: {result.stderr}") + else: + raise RuntimeError( + f"Failed to launch {runtime_id}: no free port found after " + f"{MAX_PORT_ATTEMPTS} attempts" + ) + + endpoint = f"{_container_routing_host(host)}:{host_port}" + _require_controller().redis.set(f"controller:{endpoint}:agent_id", agent_id) instance = { "agent_name": agent_name, diff --git a/ventis/deploy.py b/ventis/controller/deploy.py similarity index 98% rename from ventis/deploy.py rename to ventis/controller/deploy.py index d197342..47de634 100644 --- a/ventis/deploy.py +++ b/ventis/controller/deploy.py @@ -17,7 +17,7 @@ def my_workflow(query: str): """ try: - import ventis.ventis_context as ventis_context + import ventis.controller.ventis_context as ventis_context except ImportError: import ventis_context import json @@ -33,7 +33,7 @@ def my_workflow(query: str): # Try to import from absolute package (local install) or fallback to flat file (Docker container) try: - from ventis.utils.redis_client import RedisClient + from ventis.controller.utils.redis_client import RedisClient except ImportError: from redis_client import RedisClient diff --git a/ventis/future.py b/ventis/controller/future.py similarity index 94% rename from ventis/future.py rename to ventis/controller/future.py index 68615f1..04b050a 100644 --- a/ventis/future.py +++ b/ventis/controller/future.py @@ -1,6 +1,6 @@ import time import json -import uuid +import secrets import sys import os import logging @@ -8,12 +8,12 @@ import grpc try: - import ventis.ventis_context as ventis_context + import ventis.controller.ventis_context as ventis_context except ImportError: import ventis_context try: - from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS + from ventis.controller.utils.grpc_options import GRPC_CHANNEL_OPTIONS except ImportError: from grpc_options import GRPC_CHANNEL_OPTIONS @@ -23,7 +23,7 @@ sys.path.insert(0, os.path.abspath("grpc_stubs")) try: - from ventis.utils.redis_client import RedisClient + from ventis.controller.utils.redis_client import RedisClient except ImportError: from redis_client import RedisClient import local_controler_pb2 @@ -63,8 +63,11 @@ def __init__(self, parent, service, method, args=None): args: arguments to be passed to the method """ - # initial value of future object - self.id = uuid.uuid4().hex + # initial value of future object. 64-bit (8 bytes / 16 hex chars) -- + # this doubles as the OTel span_id (convert.py), which is defined as + # 64-bit, so it's generated at that width directly instead of a + # 128-bit uuid4 that would need truncating later. + self.id = secrets.token_hex(8) # Grab the request_id from the thread-local context (set by deploy) self.request_id = ventis_context.get_request_id() diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index fc5dd38..c85dcbf 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -13,6 +13,7 @@ import sys import threading import time +import uuid from concurrent.futures import ThreadPoolExecutor import yaml @@ -28,11 +29,14 @@ send_runtime_information, send_agent_information, ) -from ventis.utils.redis_client import RedisClient -from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS - -# Add generated grpc_stubs from the local project to the path -sys.path.insert(0, os.path.abspath("grpc_stubs")) +from ventis.controller.utils.redis_client import RedisClient +from ventis.controller.utils.grpc_options import GRPC_CHANNEL_OPTIONS + +# Add generated grpc_stubs from the local project to the path. Projects using +# the .car artifact layout keep grpc_stubs under .car/; older/plain layouts +# keep it at the project root. +_artifact_prefix = ".car" if os.path.isdir(".car") else "" +sys.path.insert(0, os.path.abspath(os.path.join(_artifact_prefix, "grpc_stubs"))) import local_controler_pb2 import local_controler_pb2_grpc import grpc @@ -92,7 +96,7 @@ def __init__(self, config_path): self._last_metrics_poll_time = {} # (host, port) -> time.time() of last metrics read self._lc_stubs = {} # endpoint -> gRPC stub self.instance_manager = InstanceManager(self) - assign_project_id(self.config.get("project_id",0)) + assign_project_id(self.config.get("project_id")) # Clean up any stale containers from previous runs self._cleanup_stale_containers() @@ -188,12 +192,27 @@ def _cleanup_stale_containers(self): def _load_config(config_path): """Load the YAML config file after importing root .env values.""" project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) + # Under the .car layout, config lives at /.car/config, so the + # naive parent-of-parent lands on .car itself -- go up one more level + # to reach the actual project root where .env lives. + if os.path.basename(project_root) == ".car": + project_root = os.path.dirname(project_root) GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: config = yaml.safe_load(f) + if not config.get("project_id"): + config["project_id"] = GlobalController._assign_new_project_id(config_path) config = GlobalController._expand_env_value(config) return config + @staticmethod + def _assign_new_project_id(config_path): + """Generate a project_id and append it to the config file so it stays stable across reloads/restarts.""" + project_id = str(uuid.uuid4()) + with open(config_path, "a") as f: + f.write(f'project_id: "{project_id}"\n') + return project_id + @staticmethod def _load_dotenv(path): """Load simple KEY=VALUE entries without overriding existing environment values.""" @@ -261,7 +280,7 @@ def reload_config(self): self.env_file_path = resolve_env_file(self.config) self.controllers = self.config.get("agents", []) self.poll_interval = self.config.get("poll_interval", 5) - assign_project_id(self.config.get("project_id", 0)) + assign_project_id(self.config.get("project_id")) self._write_identity() self.instance_manager.publish_routing_snapshot(self.controllers) @@ -324,7 +343,7 @@ def _load_and_write_policies(self): def _write_identity(self): """Publish the current project/database identity to every node's Redis.""" payload = { - "project_id": str(self.config.get("project_id", 0)), + "project_id": str(self.config.get("project_id")), "database_url": self.config.get("database", {}).get("url") or "", } targets = list(self.node_redis.values()) or [self.redis] @@ -377,8 +396,11 @@ def _launch_redis_containers(self): redis_port = node_cfg["redis_port"] user = node_cfg["user"] container_name = f"ventis-redis-{host.replace('.', '-')}" - # For localhost, connect directly; for remote, connect via host IP - connect_host = "localhost" if host in ("localhost", "127.0.0.1") else host + # VENTIS_REDIS_HOST overrides the localhost case for a containerized GC; host/remote paths unchanged. + if host in ("localhost", "127.0.0.1"): + connect_host = os.environ.get("VENTIS_REDIS_HOST", "localhost") + else: + connect_host = host if self._redis_container_healthy(container_name, host, user, connect_host, redis_port): logger.info("Reusing existing Redis container %s on %s", container_name, host) @@ -562,7 +584,7 @@ def _poll_one_instance(self, instance): try: future_rows = pull_runtime_information(node_redis) self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) + future_rows, node_redis, self.config.get("project_id") ) # This is now legacy, keeping it for now, but will remove this later send_runtime_information( @@ -893,9 +915,9 @@ def stop(self): if __name__ == "__main__": - script_dir = os.path.dirname(os.path.abspath(__file__)) - project_root = os.path.join(script_dir, "..", "..") - default_config = os.path.join(project_root, "config", "global_controller.yaml") + default_config = os.path.join( + _artifact_prefix, "config", "global_controller.yaml" + ) import argparse @@ -904,7 +926,7 @@ def stop(self): "-c", "--config", default=default_config, - help="Path to the YAML config file (default: config/global_controller.yaml)", + help=f"Path to the YAML config file (default: {default_config})", ) args = parser.parse_args() diff --git a/ventis/controller/local_controller.py b/ventis/controller/local_controller.py index 8b5942e..8d9b525 100644 --- a/ventis/controller/local_controller.py +++ b/ventis/controller/local_controller.py @@ -18,8 +18,8 @@ try: from ventis.controller.local_controller_frontend import start_server from ventis.controller.utils.gpu_metrics import read_gpu_percent - from ventis.utils.redis_client import RedisClient - from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS + from ventis.controller.utils.redis_client import RedisClient + from ventis.controller.utils.grpc_options import GRPC_CHANNEL_OPTIONS except ImportError: from gpu_metrics import read_gpu_percent from local_controller_frontend import start_server @@ -32,7 +32,7 @@ sys.path.insert(0, os.path.abspath("grpc_stubs")) try: - import ventis.ventis_context as ventis_context + import ventis.controller.ventis_context as ventis_context except ImportError: import ventis_context import local_controler_pb2 diff --git a/ventis/controller/local_controller_frontend.py b/ventis/controller/local_controller_frontend.py index a5fcc25..722bd16 100644 --- a/ventis/controller/local_controller_frontend.py +++ b/ventis/controller/local_controller_frontend.py @@ -32,7 +32,7 @@ def __init__(self, my_endpoint="unknown"): redis_host = os.environ.get("VENTIS_REDIS_HOST", "localhost") redis_port = int(os.environ.get("VENTIS_REDIS_PORT", 6379)) try: - from ventis.utils.redis_client import RedisClient + from ventis.controller.utils.redis_client import RedisClient except ImportError: from redis_client import RedisClient self.redis = RedisClient(host=redis_host, port=redis_port) @@ -152,7 +152,7 @@ def _cleanup_request(self, request_id): def start_server(port=50051, my_endpoint="unknown"): """Start the gRPC server.""" try: - from ventis.utils.grpc_options import GRPC_SERVER_OPTIONS + from ventis.controller.utils.grpc_options import GRPC_SERVER_OPTIONS except ImportError: from grpc_options import GRPC_SERVER_OPTIONS diff --git a/ventis/utils/grpc_options.py b/ventis/controller/utils/grpc_options.py similarity index 100% rename from ventis/utils/grpc_options.py rename to ventis/controller/utils/grpc_options.py diff --git a/ventis/utils/redis_client.py b/ventis/controller/utils/redis_client.py similarity index 100% rename from ventis/utils/redis_client.py rename to ventis/controller/utils/redis_client.py diff --git a/ventis/controller/utils/telemetry_logging.py b/ventis/controller/utils/telemetry_logging.py index 7d911e2..503f3e1 100644 --- a/ventis/controller/utils/telemetry_logging.py +++ b/ventis/controller/utils/telemetry_logging.py @@ -7,7 +7,7 @@ from sqlalchemy import create_engine, text from ventis.controller.utils import pricing -from ventis.utils.redis_client import RedisClient +from ventis.controller.utils.redis_client import RedisClient logger = logging.getLogger(__name__) diff --git a/ventis/ventis_context.py b/ventis/controller/ventis_context.py similarity index 100% rename from ventis/ventis_context.py rename to ventis/controller/ventis_context.py diff --git a/ventis/llm/__init__.py b/ventis/llm/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ventis/server.py b/ventis/server.py new file mode 100644 index 0000000..8df8b0f --- /dev/null +++ b/ventis/server.py @@ -0,0 +1,70 @@ +import os +import signal +import subprocess +import sys + +from flask import Flask, jsonify, request + +app = Flask("ventis-server") + +# The project files are copied here (into a named volume) by `canyonos sync` / +# `canyonos deploy`. Deploy builds and launches against this path. +WORKSPACE_DIR = "/workspace" + +_gc_process = None + + +def _gc_running(): + return _gc_process is not None and _gc_process.poll() is None + + +@app.route("/new-project", methods=["POST"]) +def new_project(): + return jsonify({"error": "new-project runs locally via the CLI"}), 400 + + +@app.route("/deploy", methods=["POST"]) +def deploy(): + global _gc_process + + if _gc_running(): + return jsonify({"error": "already running"}), 409 + + data = request.get_json(force=True, silent=True) or {} + config_path = data.get("config_path", "config/global_controller.yaml") + full_path = os.path.join(WORKSPACE_DIR, config_path) + + if not os.path.isfile(full_path): + return jsonify({"error": f"config file not found: {full_path}"}), 400 + + # `ventis deploy` builds (stubs/protos/images) then launches the Global + # Controller. cwd is the workspace so build outputs land alongside the + # project files and the controller finds them. Build+deploy output streams + # to the container logs, which `canyonos deploy` tails. + _gc_process = subprocess.Popen( + [sys.executable, "-m", "ventis.cli", "deploy", "-c", config_path], + cwd=WORKSPACE_DIR, + ) + return jsonify({"status": "started", "pid": _gc_process.pid}), 200 + + +@app.route("/clean", methods=["POST"]) +def clean(): + global _gc_process + + if not _gc_running(): + return jsonify({"error": "not running"}), 409 + + _gc_process.send_signal(signal.SIGTERM) + _gc_process.wait() + _gc_process = None + return jsonify({"status": "stopped"}), 200 + + +@app.route("/status", methods=["GET"]) +def status(): + return jsonify({"running": _gc_running()}), 200 + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8000) diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 33824ff..1108be1 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -378,8 +378,8 @@ def generate_docker( # Copy general agent files files_to_copy += [ # (source_path, destination_filename) - (os.path.join(script_dir, "future.py"), "future.py"), - (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), + (os.path.join(script_dir, "controller", "future.py"), "future.py"), + (os.path.join(script_dir, "controller", "ventis_context.py"), "ventis_context.py"), ( os.path.join(script_dir, "controller", "local_controller.py"), "local_controller.py", @@ -388,26 +388,20 @@ def generate_docker( os.path.join(script_dir, "controller", "local_controller_frontend.py"), "local_controller_frontend.py", ), - (os.path.join(script_dir, "utils", "redis_client.py"), "redis_client.py"), - (os.path.join(script_dir, "utils", "grpc_options.py"), "grpc_options.py"), + (os.path.join(script_dir, "controller", "utils", "redis_client.py"), "redis_client.py"), + (os.path.join(script_dir, "controller", "utils", "grpc_options.py"), "grpc_options.py"), ( os.path.join(script_dir, "controller", "utils", "gpu_metrics.py"), "gpu_metrics.py", ), - (os.path.join(script_dir, "llm", "bedrock.py"), "bedrock.py"), + (os.path.join(script_dir, "controller", "bedrock.py"), "bedrock.py"), ] # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: - files_to_copy.append( - ( - os.path.abspath(stub_file), - _stub_destination(stub_file, stub_entrypoints or {}), - ) - ) - - files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) + destination = _stub_destination(stub_file, stub_entrypoints or {}) + files_to_copy.append((os.path.abspath(stub_file), destination)) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -417,6 +411,12 @@ def generate_docker( _copy_files(output_dir, files_to_copy) + # Copy the real agent entrypoint to the context root. + shutil.copy2( + os.path.abspath(agent_file), + os.path.join(output_dir, os.path.basename(agent_file)), + ) + # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -500,10 +500,9 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ - (os.path.abspath(workflow_file), workflow_basename), - (os.path.join(script_dir, "future.py"), "future.py"), - (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), - (os.path.join(script_dir, "deploy.py"), "deploy.py"), + (os.path.join(script_dir, "controller", "future.py"), "future.py"), + (os.path.join(script_dir, "controller", "ventis_context.py"), "ventis_context.py"), + (os.path.join(script_dir, "controller", "deploy.py"), "deploy.py"), ( os.path.join(script_dir, "controller", "local_controller.py"), "local_controller.py", @@ -512,8 +511,8 @@ def generate_workflow_docker( os.path.join(script_dir, "controller", "local_controller_frontend.py"), "local_controller_frontend.py", ), - (os.path.join(script_dir, "utils", "redis_client.py"), "redis_client.py"), - (os.path.join(script_dir, "utils", "grpc_options.py"), "grpc_options.py"), + (os.path.join(script_dir, "controller", "utils", "redis_client.py"), "redis_client.py"), + (os.path.join(script_dir, "controller", "utils", "grpc_options.py"), "grpc_options.py"), *[ (os.path.join(script_dir, "controller", "utils", name), name) for name in ("gpu_metrics.py", "session_logging.py") @@ -522,12 +521,8 @@ def generate_workflow_docker( # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - files_to_copy.append( - ( - os.path.abspath(stub_file), - _stub_destination(stub_file, stub_entrypoints or {}), - ) - ) + destination = _stub_destination(stub_file, stub_entrypoints or {}) + files_to_copy.append((os.path.abspath(stub_file), destination)) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -537,6 +532,12 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) + # Copy the real workflow entrypoint to the context root. + shutil.copy2( + os.path.abspath(workflow_file), + os.path.join(output_dir, workflow_basename), + ) + # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time diff --git a/ventis/utils/__init__.py b/ventis/utils/__init__.py deleted file mode 100644 index 7863cb0..0000000 --- a/ventis/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# empty proxy module for packaging From 0508422fa5367d49844777cfea0aa428413da5cb Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 11:37:38 -0700 Subject: [PATCH 23/31] Point example workflow imports at the stub's entrypoint path Stub placement is now mirrored-only, so a workflow importing the flat basename no longer resolves -- the stub is written to agents/.py and nothing is left at the context root. Switch the four example workflows to the nested form. Cherry-picked from 7f925ef on fix/remove-duplicate-stub. Co-Authored-By: Claude Opus 5 (1M context) --- examples/finance/workflow/example_workflow.py | 4 ++-- examples/helloworld/workflow/example_workflow.py | 2 +- examples/portfolio/workflow/portfolio_workflow.py | 8 ++++---- examples/text2sql/workflow/text2sql_workflow.py | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/finance/workflow/example_workflow.py b/examples/finance/workflow/example_workflow.py index cac18ad..e4b6ce5 100644 --- a/examples/finance/workflow/example_workflow.py +++ b/examples/finance/workflow/example_workflow.py @@ -18,8 +18,8 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from finance_agent import FinanceAgent -from market_agent import MarketResearchAgent +from agents.finance_agent import FinanceAgent +from agents.market_agent import MarketResearchAgent def main(ticker: str = "AAPL"): diff --git a/examples/helloworld/workflow/example_workflow.py b/examples/helloworld/workflow/example_workflow.py index 842fe80..6bafff3 100644 --- a/examples/helloworld/workflow/example_workflow.py +++ b/examples/helloworld/workflow/example_workflow.py @@ -15,7 +15,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from example_agent import ExampleAgent +from agents.example_agent import ExampleAgent def main(name: str = "World"): diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 44a1484..619c4bd 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -31,10 +31,10 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from intent_agent import IntentAgent -from metrics_agent import MetricsAgent -from risk_agent import RiskAgent -from advisor_agent import AdvisorAgent +from agents.intent_agent import IntentAgent +from agents.metrics_agent import MetricsAgent +from agents.risk_agent import RiskAgent +from agents.advisor_agent import AdvisorAgent def main( diff --git a/examples/text2sql/workflow/text2sql_workflow.py b/examples/text2sql/workflow/text2sql_workflow.py index d2f32eb..ac9801d 100644 --- a/examples/text2sql/workflow/text2sql_workflow.py +++ b/examples/text2sql/workflow/text2sql_workflow.py @@ -25,11 +25,11 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from schema_agent import SchemaRetrievalAgent -from sql_generator_agent import SQLGeneratorAgent -from sql_validator_agent import SQLValidatorAgent -from sandbox_agent import SandboxExecutorAgent -from production_agent import ProductionExecutorAgent +from agents.schema_agent import SchemaRetrievalAgent +from agents.sql_generator_agent import SQLGeneratorAgent +from agents.sql_validator_agent import SQLValidatorAgent +from agents.sandbox_agent import SandboxExecutorAgent +from agents.production_agent import ProductionExecutorAgent def main(question: str = "total order amount per customer region", n_candidates: int = 3): From ff61053a171bab90417e3f19e0c28d37ad2dabc6 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 11:42:01 -0700 Subject: [PATCH 24/31] Route local-provider agents over a dedicated docker network Cherry-picked from a0efb5a on bug/local-provider-routing, resolved against the .car/CLI packaging changes already on this branch. Agents used to be reached at host.docker.internal:, which only works when whatever is doing the reaching shares the host's network namespace. They now join a `ventis-local` docker network and are addressed by container name at the fixed container port, so routing no longer depends on the caller's vantage point. - Local/_runtime.py: --network ventis-local instead of --add-host, VENTIS_AGENT_PORT is the container port, VENTIS_AGENT_HOST is the container name, routing_endpoint_for returns :50051. - global_controller.py: creates the network alongside the local Redis container, and derives status/metrics Redis keys from instance_manager._routing_endpoint_for instead of the host string. Conflict resolution notes: - Kept this branch's MAX_PORT_ATTEMPTS retry loop and env_file support around the docker run, applying the new network/env flags inside it. - Kept _is_local_host in Local/_runtime.py; a0efb5a dropped it, but env_file_args (added later on the CLI line) still needs it. - The controller::agent_id key written after launch, which postdates a0efb5a, now uses the container-name endpoint so it matches the keys global_controller reads back. Co-Authored-By: Claude Opus 5 (1M context) --- .../helloworld/config/global_controller.yaml | 6 +-- tests/test_instance_manager_runtime.py | 22 +++++----- .../cloud_provider_logic/Local/_runtime.py | 20 ++++----- ventis/controller/global_controller.py | 44 +++++++++---------- 4 files changed, 44 insertions(+), 48 deletions(-) diff --git a/examples/helloworld/config/global_controller.yaml b/examples/helloworld/config/global_controller.yaml index 0b9c194..5f6c0cc 100644 --- a/examples/helloworld/config/global_controller.yaml +++ b/examples/helloworld/config/global_controller.yaml @@ -10,7 +10,7 @@ agents: cpu: 1 memory: 512 entrypoint: agents/example_agent.py - provider: EC2 + provider: local - name: VllmAgent replicas: 1 @@ -19,7 +19,7 @@ agents: cpu: 2 memory: 2048 entrypoint: agents/vllm_agent.py - provider: EC2 + provider: local instance_type: t3.micro - name: Workflow @@ -28,7 +28,7 @@ agents: redis_port: 6379 api_port: 8080 # Only needed for workflows, defaults to 8080 if not filled workflow_file: workflow/example_workflow.py - provider: EC2 + provider: local instance_type: t3.micro poll_interval: 5 diff --git a/tests/test_instance_manager_runtime.py b/tests/test_instance_manager_runtime.py index df35efe..13f9876 100644 --- a/tests/test_instance_manager_runtime.py +++ b/tests/test_instance_manager_runtime.py @@ -138,7 +138,7 @@ def test_local_instances_keep_default_host_and_increment_host_ports(self): "host_port": "8000", "container_port": "50051", "endpoint": "localhost:8000", - "redis_host": "host.docker.internal", + "redis_host": "ventis-redis-localhost", "redis_port": "6379", "runtime_id": "ventis-local-alpha-0", }, @@ -154,17 +154,18 @@ def test_local_instances_keep_default_host_and_increment_host_ports(self): "run", "-d", "-it", - "--add-host=host.docker.internal:host-gateway", + "--network", + "ventis-local", "--name", "ventis-local-alpha-0", "-p", "8000:50051", "-e", - "VENTIS_AGENT_PORT=8000", + "VENTIS_AGENT_PORT=50051", "-e", - "VENTIS_AGENT_HOST=host.docker.internal", + "VENTIS_AGENT_HOST=ventis-local-alpha-0", "-e", - "VENTIS_REDIS_HOST=host.docker.internal", + "VENTIS_REDIS_HOST=ventis-redis-localhost", "-e", "VENTIS_REDIS_PORT=6379", "-e", @@ -209,17 +210,18 @@ def test_local_workflow_and_resource_flags_stay_the_same(self): "run", "-d", "-it", - "--add-host=host.docker.internal:host-gateway", + "--network", + "ventis-local", "--name", "ventis-local-workflow-0", "-p", "8000:50051", "-e", - "VENTIS_AGENT_PORT=8000", + "VENTIS_AGENT_PORT=50051", "-e", - "VENTIS_AGENT_HOST=host.docker.internal", + "VENTIS_AGENT_HOST=ventis-local-workflow-0", "-e", - "VENTIS_REDIS_HOST=host.docker.internal", + "VENTIS_REDIS_HOST=ventis-redis-localhost", "-e", "VENTIS_REDIS_PORT=6379", "-e", @@ -255,7 +257,7 @@ def test_agent_id_is_published_under_the_controller_endpoint_key(self): alpha = manager.ensure_instances([{"name": "Alpha", "provider": "local"}])[0] self.assertEqual( - controller.redis.get("controller:host.docker.internal:8000:agent_id"), + controller.redis.get("controller:ventis-local-alpha-0:50051:agent_id"), alpha["agent_id"], ) diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index 1a84e56..dda4807 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -16,6 +16,7 @@ CONTAINER_PORT = 50051 PROVIDER = "local" MAX_PORT_ATTEMPTS = 50 +NETWORK = "ventis-local" _controller = None @@ -29,10 +30,6 @@ def _is_local_host(host): return host in {"localhost", "127.0.0.1"} -def _container_routing_host(host): - return "host.docker.internal" if _is_local_host(host) else host - - def validate_config(): return None @@ -46,7 +43,7 @@ def provision_instance(spec, replica_index, next_host_port): "provider": PROVIDER, "host": host, "host_port": host_port, - "redis_host": _container_routing_host(host), + "redis_host": f"ventis-redis-{host.replace('.', '-')}", "runtime_id": f"ventis-{PROVIDER}-{agent_name.lower()}-{replica_index}", "user": spec.get("user"), } @@ -80,15 +77,16 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): "run", "-d", "-it", - "--add-host=host.docker.internal:host-gateway", + "--network", + NETWORK, "--name", runtime_id, "-p", f"{host_port}:{CONTAINER_PORT}", "-e", - f"VENTIS_AGENT_PORT={host_port}", + f"VENTIS_AGENT_PORT={CONTAINER_PORT}", "-e", - f"VENTIS_AGENT_HOST={redis_host}", + f"VENTIS_AGENT_HOST={runtime_id}", "-e", f"VENTIS_REDIS_HOST={redis_host}", "-e", @@ -138,7 +136,7 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): f"{MAX_PORT_ATTEMPTS} attempts" ) - endpoint = f"{_container_routing_host(host)}:{host_port}" + endpoint = f"{runtime_id}:{CONTAINER_PORT}" _require_controller().redis.set(f"controller:{endpoint}:agent_id", agent_id) instance = { @@ -174,6 +172,4 @@ def terminate_instance(instance): def routing_endpoint_for(instance): - host = instance.get("host") - port = instance["host_port"] - return f"{_container_routing_host(host)}:{port}" + return f"{instance['runtime_id']}:{CONTAINER_PORT}" diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index c85dcbf..4bf109a 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -44,15 +44,13 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +LOCAL_NETWORK = "ventis-local" + def _is_local_host(host): return host in {"localhost", "127.0.0.1"} -def _container_routing_host(host): - return "host.docker.internal" if _is_local_host(host) else host - - class GlobalController(object): """ Daemon that manages a routing table across multiple local controller instances. @@ -406,12 +404,16 @@ def _launch_redis_containers(self): logger.info("Reusing existing Redis container %s on %s", container_name, host) self.redis_containers[host] = container_name else: + if _is_local_host(host): + self._run_cmd(["docker", "network", "create", LOCAL_NETWORK], host, user) + network_args = ["--network", LOCAL_NETWORK] if _is_local_host(host) else [] cmd = [ "docker", "run", "-d", "--name", container_name, + *network_args, "-p", f"{redis_port}:6379", "redis:alpine", @@ -484,10 +486,6 @@ def _get_node_redis_for(self, host): """Get the Redis client for a given host, falling back to self.redis.""" return self.node_redis.get(host, self.redis) - def _agent_host_key(self, host): - """Return the host string as seen by Docker containers (for status key matching).""" - return _container_routing_host(host) - def _wait_for_healthy(self, timeout=30, interval=2): """ Block until all controllers report healthy in Redis, or until timeout. @@ -497,10 +495,7 @@ def _wait_for_healthy(self, timeout=30, interval=2): interval: Seconds between checks. """ deadline = time.time() + timeout - pending = [ - (instance["agent_name"], instance["host"], instance["host_port"]) - for instance in self.instance_manager.list_instances() - ] + pending = self.instance_manager.list_instances() logger.info( "Waiting for %d replica(s) to become healthy (timeout=%ds)...", @@ -510,26 +505,29 @@ def _wait_for_healthy(self, timeout=30, interval=2): while pending and time.time() < deadline: still_pending = [] - for name, host, port in pending: + for instance in pending: + name = instance["agent_name"] + host = instance["host"] + port = instance["host_port"] node_redis = self._get_node_redis_for(host) - agent_host = self._agent_host_key(host) - status = node_redis.get(f"controller:{agent_host}:{port}:status") + endpoint = self.instance_manager._routing_endpoint_for(instance) + status = node_redis.get(f"controller:{endpoint}:status") if status == "healthy": logger.info("Controller %s (%s:%s) is ready.", name, host, port) self._last_status[(host, port)] = "healthy" else: - still_pending.append((name, host, port)) + still_pending.append(instance) pending = still_pending if pending: time.sleep(interval) if pending: - for name, host, port in pending: + for instance in pending: logger.warning( "Controller %s (%s:%s) not ready after %ds.", - name, - host, - port, + instance["agent_name"], + instance["host"], + instance["host_port"], timeout, ) @@ -601,9 +599,9 @@ def _poll_one_instance(self, instance): port, e, ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + endpoint = self.instance_manager._routing_endpoint_for(instance) + status_key = f"controller:{endpoint}:status" + metrics_key = f"controller:{endpoint}:metrics" # Getting metrics from local controllers # See LocalController._execute_locally From 6f36d4f1c090def778f5275c8b25da45a9844864 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 12:22:28 -0700 Subject: [PATCH 25/31] Add the canyonos CLI package, porting skill, and .car examples The CLI layer on top of the ventis fixes: `canyonos` wraps the Global Controller in a container and drives it over HTTP, so a project goes from source to a running workflow with a local dashboard in one command. - cli/: the canyonos package -- deploy (which folds in init, sync, build and launch, then auto-starts the dashboard once the workflow reports up), serve, stop, logs, quit, clean, config, integrate, new-app. - cli/canyonos/dashboard.compose.yml: api/db/web stack. The api port is published so the GC container can POST OTLP spans to /v1/traces, which is also what renames ventis' `project_id` attribute to the `canyon.project.id` the dashboard queries filter on. serve replaces the api container every run, since it reads the controller's Redis identity only at startup and would otherwise keep serving a stale project. - cli/canyonos/constants.py: resolve the config path per call rather than at import, preferring .car/config over the flat layout. - .claude/skills/porting-to-canyonos-core/: the porting skill `canyonos integrate` installs, plus its validator. - examples/: joke_writer converted to the .car layout, epigenomics added, and workflow imports pointed at each agent's entrypoint path to match single-location stub placement. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 4 + README.md | 32 +- SESSION_NOTES.md | 79 ++ .../skills/porting-to-canyonos-core/SKILL.md | 240 ++++ .../references/ec2.md | 41 + .../references/llm-proxy.md | 64 + .../references/packaging.md | 81 ++ .../references/runtime-contract.md | 187 +++ .../references/troubleshooting.md | 60 + .../porting-to-canyonos-core/validate.py | 1257 +++++++++++++++++ cli/README.md | 21 + cli/canyonos/__init__.py | 0 cli/canyonos/clean.py | 28 + cli/canyonos/config.py | 345 +++++ cli/canyonos/constants.py | 9 + cli/canyonos/dashboard.compose.yml | 42 + cli/canyonos/dashboard_stack.py | 518 +++++++ cli/canyonos/deploy.py | 86 ++ cli/canyonos/init.py | 129 ++ cli/canyonos/integrate.py | 82 ++ cli/canyonos/logs.py | 37 + cli/canyonos/new_app.py | 21 + cli/canyonos/quit.py | 63 + cli/canyonos/serve.py | 18 + cli/canyonos/stop.py | 47 + cli/canyonos/sync.py | 40 + cli/canyonos/theme.py | 20 + cli/cli.py | 217 +++ cli/pyproject.toml | 27 + cli/tests/test_dashboard_stack.py | 432 ++++++ cli/utils/__init__.py | 0 cli/utils/tui.py | 113 ++ examples/epigenomics/README.md | 70 + examples/epigenomics/agents/dedup_agent.py | 44 + examples/epigenomics/agents/dedup_agent.yaml | 10 + examples/epigenomics/agents/filter_agent.py | 32 + examples/epigenomics/agents/filter_agent.yaml | 12 + examples/epigenomics/agents/index_agent.py | 30 + examples/epigenomics/agents/index_agent.yaml | 12 + examples/epigenomics/agents/map_agent.py | 33 + examples/epigenomics/agents/map_agent.yaml | 14 + examples/epigenomics/agents/sort_agent.py | 32 + examples/epigenomics/agents/sort_agent.yaml | 14 + examples/epigenomics/agents/split_agent.py | 27 + examples/epigenomics/agents/split_agent.yaml | 12 + .../epigenomics/config/global_controller.yaml | 76 + examples/epigenomics/config/policy.yaml | 20 + .../workflow/epigenomics_workflow.py | 97 ++ .../helloworld/config/global_controller.yaml | 6 +- examples/joke_writer/.car/app/.env.example | 20 + examples/joke_writer/.car/app/LICENSE | 21 + examples/joke_writer/.car/app/README.md | 177 +++ .../joke_writer/.car/app/joke_workflow.py | 39 + examples/joke_writer/.car/app/joke_writer.py | 164 +++ .../.car/config/global_controller.yaml | 52 + .../joke_writer/.car/config/joke_agent.yaml | 35 + examples/joke_writer/README.md | 8 +- examples/joke_writer/agents/joke_agent.py | 63 - examples/joke_writer/agents/joke_agent.yaml | 53 - .../joke_writer/config/global_controller.yaml | 82 -- examples/joke_writer/config/policy.yaml | 20 - .../joke_writer/workflow/joke_workflow.py | 59 - .../skills/porting-to-canyonos-core/SKILL.md | 240 ++++ .../references/ec2.md | 41 + .../references/llm-proxy.md | 64 + .../references/packaging.md | 81 ++ .../references/runtime-contract.md | 187 +++ .../references/troubleshooting.md | 60 + .../porting-to-canyonos-core/validate.py | 1257 +++++++++++++++++ examples/portfolio/agents/advisor_agent.py | 4 +- examples/portfolio/agents/intent_agent.py | 4 +- .../portfolio/config/global_controller.yaml | 32 +- .../portfolio/workflow/portfolio_workflow.py | 2 +- examples/text2sql/agents/vllm_agent.py | 4 +- 74 files changed, 7321 insertions(+), 329 deletions(-) create mode 100644 SESSION_NOTES.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/SKILL.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/ec2.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/packaging.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md create mode 100755 cli/.claude/skills/porting-to-canyonos-core/validate.py create mode 100644 cli/README.md create mode 100644 cli/canyonos/__init__.py create mode 100644 cli/canyonos/clean.py create mode 100644 cli/canyonos/config.py create mode 100644 cli/canyonos/constants.py create mode 100644 cli/canyonos/dashboard.compose.yml create mode 100644 cli/canyonos/dashboard_stack.py create mode 100644 cli/canyonos/deploy.py create mode 100644 cli/canyonos/init.py create mode 100644 cli/canyonos/integrate.py create mode 100644 cli/canyonos/logs.py create mode 100644 cli/canyonos/new_app.py create mode 100644 cli/canyonos/quit.py create mode 100644 cli/canyonos/serve.py create mode 100644 cli/canyonos/stop.py create mode 100644 cli/canyonos/sync.py create mode 100644 cli/canyonos/theme.py create mode 100644 cli/cli.py create mode 100644 cli/pyproject.toml create mode 100644 cli/tests/test_dashboard_stack.py create mode 100644 cli/utils/__init__.py create mode 100644 cli/utils/tui.py create mode 100644 examples/epigenomics/README.md create mode 100644 examples/epigenomics/agents/dedup_agent.py create mode 100644 examples/epigenomics/agents/dedup_agent.yaml create mode 100644 examples/epigenomics/agents/filter_agent.py create mode 100644 examples/epigenomics/agents/filter_agent.yaml create mode 100644 examples/epigenomics/agents/index_agent.py create mode 100644 examples/epigenomics/agents/index_agent.yaml create mode 100644 examples/epigenomics/agents/map_agent.py create mode 100644 examples/epigenomics/agents/map_agent.yaml create mode 100644 examples/epigenomics/agents/sort_agent.py create mode 100644 examples/epigenomics/agents/sort_agent.yaml create mode 100644 examples/epigenomics/agents/split_agent.py create mode 100644 examples/epigenomics/agents/split_agent.yaml create mode 100644 examples/epigenomics/config/global_controller.yaml create mode 100644 examples/epigenomics/config/policy.yaml create mode 100644 examples/epigenomics/workflow/epigenomics_workflow.py create mode 100644 examples/joke_writer/.car/app/.env.example create mode 100644 examples/joke_writer/.car/app/LICENSE create mode 100644 examples/joke_writer/.car/app/README.md create mode 100644 examples/joke_writer/.car/app/joke_workflow.py create mode 100644 examples/joke_writer/.car/app/joke_writer.py create mode 100644 examples/joke_writer/.car/config/global_controller.yaml create mode 100644 examples/joke_writer/.car/config/joke_agent.yaml delete mode 100644 examples/joke_writer/agents/joke_agent.py delete mode 100644 examples/joke_writer/agents/joke_agent.yaml delete mode 100644 examples/joke_writer/config/global_controller.yaml delete mode 100644 examples/joke_writer/config/policy.yaml delete mode 100644 examples/joke_writer/workflow/joke_workflow.py create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md create mode 100755 examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py diff --git a/.gitignore b/.gitignore index 085b4a3..1d7998b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ env/ Thumbs.db ._* +# Canyon artifacts. `.car` is generated from the application source by the +# porting skill and `ventis build`; it is never committed. +.car/ + # Generated stubs stubs/ grpc_stubs/ diff --git a/README.md b/README.md index 81d944f..1d61db8 100644 --- a/README.md +++ b/README.md @@ -39,26 +39,26 @@ cd my-app ``` This command creates a new directory `my-app` with the following structure: ``` -├── agents/ # Agent implementations and YAML definitions -│ ├── example_agent.py -│ └── example_agent.yaml -├── workflows/ # Workflow scripts (deployed as REST APIs) -│ └── example_workflow.py -├── config/ -│ ├── global_controller.yaml # Deployment configuration -│ └── policy.yaml # Access control rules -├── stubs/ # Generated agent stubs (auto-generated) -├── grpc_stubs/ # Generated gRPC stubs (auto-generated) -└── README.md # Readme for the project +├── .car/ +│ ├── app/ # Source copy used for builds +│ ├── config/ +│ │ ├── global_controller.yaml +│ │ ├── example_agent.yaml # Agent declaration +│ │ └── policy.yaml +│ ├── stubs/ +│ ├── grpc_stubs/ +│ └── docker_container/ +└── README.md ``` The Readme in the newly created project directory provides a quick overview of the project and how to use it. Including how to add new files etc. We provide some overview in next few steps. #### Step 2: Define Your Agents -Place your agent logic (`.py`) and definitions (`.yaml`) in the `agents/` directory. +Agent declarations live under `.car/config/`. The source used for builds is +copied to `.car/app/` by `canyonos integrate`. -- **`agents/my_agent.yaml`**: Defines methods and schemas. -- **`agents/my_agent.py`**: Contains the actual Python implementation. +- **`.car/config/my_agent.yaml`**: Defines methods and schemas. +- **`.car/app/path/to/my_agent.py`**: Contains the agent implementation. We have provided an example of a finance agent and a market research agent in the `examples/` directory. To run the example, copy files into your newly created project directory from within the your my-app directory with the command - @@ -69,14 +69,14 @@ cp -r ../examples/* ./ ## Deployment Guide #### Step 1: Configure the Global Controller -Edit `config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. +Edit `.car/config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. #### Step 1.1: Passing secrets to agents (optional) Agents that need API keys read them from environment variables. Point `env_file` at a `.env` file to have Ventis inject it into every agent container: ```yaml -# config/global_controller.yaml +# .car/config/global_controller.yaml env_file: .env ``` diff --git a/SESSION_NOTES.md b/SESSION_NOTES.md new file mode 100644 index 0000000..360efcc --- /dev/null +++ b/SESSION_NOTES.md @@ -0,0 +1,79 @@ +# Session notes: canyonos serve, OTel pipeline, examples + +## Fixed (code changes, rebuilt+pushed `saakeths/canyonos:latest` where needed) + +1. **`ventis/stub_generator.py`**: stubs only ever got placed at ONE path + (nested-at-entrypoint, never flat), breaking any workflow that imports a + sibling agent directly (`from split_agent import SplitAgent`, e.g. + `examples/epigenomics`). Now placed at both. +2. **`otel_exporter.py`**: hardcoded Redis to `localhost`, but it runs inside + the GC container (bridge networking) while Redis is a sibling container — + crash-looped forever. Fixed to `host.docker.internal`. +3. **`ventis/OTLP_Exporter/db.py`**: `write_waiting_rows()` unconditionally + priced every future via a `aws_instance_pricing` table that only exists for + EC2 deployments — silently dropped **every** span for **every** + `provider: local` deployment, always. Wrapped cost lookups in try/except, + falls back to $0. +4. **`dashboard_stack.py`**: `web` port was hardcoded 8080 with no fallback. + Now searches for a free port (reuses an already-running dashboard's port + if one exists), same pattern as `init.py`'s GC port selection. +5. **`dashboard_stack.py`**: `database.url` was required; made optional + (dashboard boots fine with no DB configured in the project's own config). +6. **`dashboard.compose.yml`**: added `pull_policy: never` for the local-only + `canyonos-otel-receiver:local` image (compose was trying to pull it from a + registry that doesn't have it). Sequenced `otel-receiver` to start after + `api` (both raced to create `otel_spans`; `api`'s bare `CREATE TABLE` lost + the race and crashed). + +## Bundled (new, working) + +- `db` (plain Postgres) + `otel-receiver` (new `Dockerfile` for + `otlp_pg_receiver`, built locally as `canyonos-otel-receiver:local`) added + to `dashboard.compose.yml`. Verified real spans, sent via the actual OTLP + gRPC exporter, land in `otel_spans`. + +## Workarounds applied, NOT real fixes (will resurface) + +- **`canyonos quit` only tears down the GC container + volume**, never the + deployed agent/workflow/redis containers. Had to `docker rm -f` those by + exact name every time before a truly clean restart. +- **The named workspace volume is additive-only** (`docker cp`, never + clears) — files from a previous project leak into the next one's build + until you manually nuke the volume. +- **`otlp_pg_receiver` holds one Postgres connection with no reconnect + logic** — a DB restart silently kills every future write until the + receiver container itself is restarted. +- **`joke_writer/.car` layout** (`app/`, `config/`) doesn't match what + `ventis/cli.py`'s build step expects (`agents/`, flat entrypoints) — worked + around by manually copying files into the shape it wants. The real fix + (`nickhuo/car-artifact-layout`, already pushed) was not merged in. +- **joke_writer's LLM calls are stubbed to return `"animal"`** — for testing + only, real Bedrock creds needed to restore actual behavior (commented-out + code left in place). + +## Known, not touched + +- Pre-existing OrbStack local-provider startup race (first request right + after a container reports healthy can fail); a fix exists on an unrelated, + unmerged branch. +- Stale global `uv tool install` is a recurring trap — always + `uv tool install --reinstall .` after any `cli/` change. + +## What's still needed to actually see data in the UI + +The whole pipeline up to Postgres now genuinely works. **Nothing shows up in +the dashboard because `canyonos-api`/`canyonos-web` have zero code that reads +or displays `otel_spans`** — confirmed by inspecting their actual source +(they're a `cc-forge` rebrand: deploy/project management + a static +code-structure diagram, unrelated data model). To close the loop: + +1. New API route(s) in `canyon-code-forge/packages/api` that query + `otel_spans`. +2. New UI screen(s) in `canyon-code-forge/apps/web` to render it. +3. `web` currently has **no path to reach `api` at all** even once that + exists — no reverse proxy in its Caddyfile, and `api`'s port isn't + published to the host in `dashboard.compose.yml`. Needs one or the other + before the browser can fetch anything. + +All of the above is real feature work in a different repo, not a config or +wiring fix. diff --git a/cli/.claude/skills/porting-to-canyonos-core/SKILL.md b/cli/.claude/skills/porting-to-canyonos-core/SKILL.md new file mode 100644 index 0000000..fe6dd49 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/SKILL.md @@ -0,0 +1,240 @@ +--- +name: porting-to-canyonos-core +description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. +compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. +--- + +# Port an agent project to CanyonOS Core + +CanyonOS Core is the product name. Its compatibility executable and Python +package remain `ventis`; environment variables and Docker resources retain the +`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding +strings. Do not rename them. + +## Load references only when needed + +- Read [references/packaging.md](references/packaging.md) when a source import + does not resolve from `/app`, the source is nested, or packaging metadata is + involved. +- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target + includes `llm_proxy`. +- Read [references/ec2.md](references/ec2.md) only when any config entry uses + `provider: EC2`. +- Read [references/troubleshooting.md](references/troubleshooting.md) after a + failed build, image probe, deploy, or request. +- Read [references/runtime-contract.md](references/runtime-contract.md) when a + validator finding needs explanation or the runtime mechanism is unclear. + +## Goal: thin scaffolding beside untouched source + +```text +agents/.yaml one callable surface per service +agents/.py one thin adapter per service, when needed +workflow/_workflow.py HTTP entry point; calls deploy() +config/global_controller.yaml deployment manifest +config/policy.yaml optional access restriction +pyproject.toml conditional nested-import scaffolding + unchanged +``` + +The file count follows the deployment. A multi-agent port has one yaml/adapter +pair per service that is worth deploying separately. If a source class already +satisfies the runtime contract, point its config entry at that file and do not +copy it into an adapter. + +Everything the source already owns—prompts, tools, schemas, parsing, retries, +model clients, and node bodies—is imported. The port re-expresses only the +CanyonOS Core boundary and framework-owned orchestration. + +The port root is the existing repository root and the directory from which +`ventis build` runs. Write scaffolding there beside existing directories. If the +repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, +and `config/` beside it. Never move or copy the repository into a new `src/` +directory, and never create an outer wrapper merely for the port. + +## 1. Survey before writing + +Identify: + +1. The source entry point and callable input/output. +2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, + `Send`, `Command`, interrupts). +3. Runtime-injected services nodes read: stores, context, memory, sessions, or + callback managers. +4. Sync versus async boundaries. +5. Imports and declared runtime dependencies. +6. Model provider, credential names, streaming use, and optional `llm_proxy`. +7. Whether independent work fans out and benefits from separate replicas. +8. Whether source imports resolve from the project root that becomes `/app`. + +Run the validator once now. Its header detects capabilities directly from the +importable runtime rather than external development metadata: + +```bash +python /validate.py . +``` + +If config or agent yaml is malformed, the validator defers to `ventis build`. +Capability-gated findings say which runtime behavior is available. + +## 2. Choose service boundaries + +Start with one service. Split only when it creates independent parallel work or +a distinct resource/replica profile. + +- Keep a ReAct loop together; every turn needs shared message history. +- Hoist supervisor task lists and `Send`-style fan-out into the workflow. +- Do not create a one-replica service with no distinct resource profile merely + to mirror every source graph node. + +Rewrite framework-owned edges as ordinary Python. Import the connected node +functions unchanged. Construct runtime-injected service objects from source +configuration; do not invent models, dimensions, stores, or defaults silently. +Report any choice the source does not specify. + +## 3. Write declarations and adapters + +### Agent yaml + +Use one yaml per deployed service. Argument types are bare builtins only: +`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is +required by the generated stub. `returns.type` is documentation; use `dict` or +`list` to signal that workflow callers must `json.loads` the returned string. + +### Adapter + +The entrypoint exposes a module-level class named exactly `agent.name`. It +constructs with no arguments and its declared methods are synchronous. Read +configuration from the environment in `__init__`. Bridge source coroutines +inside a synchronous method with `asyncio.run(...)`. Serialize framework objects +with their own JSON-safe serializer before returning. + +Do not duplicate source prompts, tools, schemas, or model calls. Keep the source +provider and SDK. + +### Workflow + +Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. +Import generated stubs by yaml basename and agent class name: + +```python +from deploy import deploy +from agents. import +``` + +The deployment platform sends `{query: string}` to `/main`. Pack richer input +inside `query`; any additional workflow parameter has a default. + +Dispatch every remote call before resolving any future: + +```python +futures = [agent.work(item=item) for item in items] +results = [json.loads(future.value()) for future in futures] +``` + +Do not fuse dispatch and `.value()` in one comprehension; that silently +serializes fan-out. Do not add an `if __name__ == "__main__":` block: the +workflow is executed with `__name__ == "__main__"` in production. + +### Config + +For each service, keep these names aligned: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a +list of distribution-name strings. Put `env_file` at config top level when the +runtime capability is available. Omit `policy.yaml` unless access must be +restricted; if present, give it a non-empty `rules` list. + +## Hard rules + +Capitalized **MUST** and **NEVER** are reserved for port-breaking or +source-integrity rules. The owner column states where each is decided. + +| ID | Rule | Owner | +|---|---|---| +| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | +| M2 | The class MUST construct with no arguments | V007 | +| M3 | yaml argument names MUST match Python parameter names | V008 | +| M4 | yaml argument types MUST be bare builtins | V010 | +| M5 | Declared adapter methods MUST be synchronous | V009 | +| M6 | Config names MUST match yaml agent names | build | +| M7 | Config names MUST not collide after lowercase normalization | build output | +| M8 | Local provider MUST be lowercase `local` | deploy preflight | +| M9 | `replicas` MUST be an integer | deploy preflight | +| M10 | `requirements` MUST be a list of strings | build | +| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | +| M12 | Workflow MUST NEVER contain a main guard | V017 | +| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | +| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | +| M15 | Workflow MUST import stubs from `agents.` | V023 | +| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | +| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | +| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | +| M19 | NEVER hardcode or bake a real credential into an image | W003 | +| M20 | NEVER edit or vendor the source tree | `git status` | +| M21 | NEVER swap the source LLM provider | review | +| M22 | NEVER silently move, drop, or reclassify source dependencies | review | +| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | +| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | + +## 4. Validate, build, and probe + +Run static preflight, then let the build own build-time validation: + +```bash +python /validate.py . +ventis build -c config/global_controller.yaml +``` + +A green build never imports the adapter. Probe each agent image in this order: + +```bash +# Runtime startup path + docker run --rm ventis- \ + python -c "import local_controller" + +# Agent load path; include --env-file when configured + docker run --rm --env-file ventis- \ + python -c "import importlib.util,sys; \ +s=importlib.util.spec_from_file_location('m','.py'); \ +m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ +m.();print('ok')" +``` + +Also probe the workflow image with `python -c "import local_controller"`; it has +its own dependency resolve and generated-stub imports. + +Then deploy, send a representative request, and poll its status: + +```bash +ventis deploy -c config/global_controller.yaml +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query":""}' +curl http://localhost:8080/status/ +``` + +A successful outer request with a source-level failure still proves the port +reached and returned the source behavior. Record the distinction. + +## 5. Clean up + +After collecting evidence, stop foreground deploy with Ctrl+C and wait for +controller cleanup. Remove exact leftovers if startup crashed. Then remove build +products and exact images from this config: + +```bash +ventis clean +docker image rm ventis- \ + ventis- + +test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container +docker ps -a --format '{{.Names}}' +``` + +`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it +does not remove containers or images. Keep port scaffolding, untouched source, +and requested logs or reports. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md b/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md new file mode 100644 index 0000000..e06daa0 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md @@ -0,0 +1,41 @@ +# EC2 deployment + +Read this only when at least one config entry uses `provider: EC2`. + +## Configuration + +Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The +top-level `ec2` block supplies the runtime's required infrastructure and SSH +settings. Read the target checkout's deploy preflight and EC2 runtime before +writing the block; do not copy values from an example environment. + +Typical required categories are: + +- AMI and instance type +- region and subnet +- security groups +- SSH user and credentials accepted by the runtime + +`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof +that provisioning, SSH, image transfer, or remote container startup works. + +## Networking + +A remote container's `host.docker.internal` names its own EC2 Docker host. It +does not name the local controller machine. Databases, model proxies, and other +services must use addresses reachable from every selected host. + +The environment file may be copied temporarily to a remote host by runtimes that +expose the `env_file` capability. Confirm behavior from the capability probe and +target runtime rather than assuming local Docker semantics. + +## Probes and cleanup + +Run the same runtime and adapter probes against the exact image before remote +deployment. After deploy, verify the remote container logs; controller health +can be green even when agent loading failed. + +Stop foreground deploy normally so the controller can terminate recorded EC2 +instances. If provisioning or startup fails before an instance is recorded, +inspect the cloud provider directly and remove exact leaked resources. Never use +a broad cleanup command against unrelated instances. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md new file mode 100644 index 0000000..f726bd7 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md @@ -0,0 +1,64 @@ +# LLM proxy integration + +Read this only when the target checkout contains `llm_proxy` or the deployment +explicitly routes model SDKs through it. + +## Preserve provider protocols + +The proxy redirects provider endpoints; it does not convert providers. Keep the +source SDK, model ID, request body, and response parsing unchanged. + +Configure only the provider variables the source uses: + +```dotenv +OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 +ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic +AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock +``` + +Some SDKs refuse to initialize without caller credentials. Give agent containers +non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS +credentials in the separate proxy process, not in the port's `env_file`. + +## Start locally + +The proxy defaults conflict with a typical deployment: host loopback is not +reachable from a container, and port 8080 is normally used by the workflow API. +Use a non-loopback bind and a different port: + +```bash +PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy +curl http://127.0.0.1:8081/healthz +``` + +Local CanyonOS Core containers resolve `host.docker.internal` through their +Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the +machine running `ventis deploy`. Distributed deployments need a reachable proxy +address or one proxy on each host. + +## Supported call shape + +The implementation buffers complete requests and responses: + +- OpenAI and Anthropic non-streaming HTTP calls are forwarded. +- Bedrock `invoke` is reissued through the proxy's boto3 client. +- OpenAI/Anthropic streaming is unsupported. +- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are + unsupported. + +Survey the source before selecting the proxy. Do not silently disable streaming; +report the unsupported behavior and stop. + +## Credential behavior + +- The OpenAI adapter removes caller authorization and inserts the proxy key. +- The Anthropic adapter removes caller key headers and inserts the proxy key. +- Botocore still signs requests sent to a custom endpoint, so a caller may need + placeholder AWS credentials even though the proxy reissues upstream with its + own identity. +- `/healthz` proves provider registration and Flask availability, not upstream + credential validity. + +OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return +JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed +with the upstream status and are not byte-for-byte passthrough. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md b/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md new file mode 100644 index 0000000..8520c4a --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md @@ -0,0 +1,81 @@ +# Packaging and import roots + +Read this reference when an adapter imports nested source code, the source uses a +`src/` layout, or V031 reports an import-root problem. + +## What `/app` can import + +CanyonOS Core preserves project-relative paths in the image and starts Python at +`/app`. Without an editable install, Python resolves names rooted there: + +- `/app/tools.py` as `import tools` +- `/app/pkg/__init__.py` as `import pkg` +- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace + directories without `__init__.py` + +It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become +an import root first. + +## Detect support, do not infer it from release history + +Run: + +```bash +python /validate.py . +``` + +Read the `editable_install` capability. If it is unavailable and the original +import cannot resolve from `/app`, report a runtime capability blocker and stop. +Do not add a `sys.path` hack or relocate source files. + +## Root metadata is the trigger + +When editable install is supported, only packaging metadata at the **port root** +triggers `pip install -e .`: + +```text +port-root/pyproject.toml detected +port-root/source/pyproject.toml ignored as an install trigger +``` + +A nested source repository may remain untouched. Add minimal root scaffolding +that points package discovery at the existing source package: + +```toml +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "canyonos-port" +version = "0.0.0" +dependencies = [] + +[tool.setuptools.packages.find] +where = ["source/src"] +include = ["pkg*"] +namespaces = true +``` + +Set `where` and `include` from the actual tree and original import spelling. Do +not reference a README or license from this wrapper metadata; file sweeps differ +by runtime capability and a missing referenced file makes the image build fail. + +## Dependencies in nested metadata + +A nested `pyproject.toml` is not installed merely because its Python files are +copied. Keep the source declaration unchanged and repeat its runtime +distributions in each relevant config entry's `requirements` list. This is +compatibility scaffolding, not permission to drop, move, or reclassify declared +dependencies. + +If source metadata is already at the port root, do not create a wrapper. Its +project dependencies participate in the same resolver as config requirements. +Report declared-but-unused toolchain dependencies and their image cost; let the +owner decide whether source metadata should change. + +## Validation boundary + +`ventis build` owns packaging syntax and installation errors. `validate.py` +checks only whether adapter imports appear to require a nested root that the +runtime will not expose. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md new file mode 100644 index 0000000..94f7401 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md @@ -0,0 +1,187 @@ +# CanyonOS Core runtime contract + +The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the +CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for +Docker resources. + +Read this reference when implementing an adapter or explaining a validator +finding. Runtime-dependent behavior is expressed as capabilities; run +`validate.py` against the target environment instead of inferring support from +release history. + +## Project root and discovery + +`ventis build` uses the current working directory as the project root. + +| Input | Discovery | +|---|---| +| `agents/*.yaml` | direct yaml glob under the project root | +| `config/global_controller.yaml` | default config, overridable with `-c` | +| workflow | `workflow_file` on a `type: workflow` entry | +| policy | `policy.yaml` beside the selected config file | +| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | + +The config name, yaml `agent.name`, and entrypoint class name form one binding: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +A missing match may skip an image while the command continues, so inspect build +output and generated image tags. + +## Agent yaml and generated stubs + +The consumed yaml shape is: + +```yaml +agent: + name: ExampleAgent + functions: + - name: work + arguments: + - name: query + type: str + returns: + type: dict +``` + +Argument annotations are generated from bare names without adding imports. Use +builtins. Generated methods have no defaults, so every declared argument is +required at the stub call site. `returns` does not control runtime conversion; +it documents whether workflow code should parse the returned string. + +Stub destinations differ by runtime capability and entrypoint layout. Workflow +code in this port convention imports the generated class from +`agents.`. The validator checks that import against declarations +before the workflow image starts. + +## Agent loading and execution + +The local controller effectively performs: + +```python +module = load(entrypoint) +agent_class = getattr(module, configured_name) +agent = agent_class() +result = getattr(agent, method_name)(**args) +``` + +Consequences: + +- The class is module-level and named exactly as configured. +- Construction takes no arguments. +- Declared methods accept yaml argument names as keyword arguments. +- Methods are synchronous; this path does not await a coroutine. +- Dicts and lists are JSON-encoded before entering Redis; other results become + strings. +- A remote Future's `.value()` returns text, not the original Python object. + +Agent import and construction exceptions are caught by the controller. A failed +agent may still advertise healthy because health is written independently of +successful agent loading. That is why image probes import both the runtime and +the entrypoint explicitly. + +## Workflow execution + +The workflow file is executed, not imported. Therefore: + +- module-level code runs at container startup; +- `__name__ == "__main__"`; +- `deploy()` blocks in the web server; +- the workflow function runs once per request; +- its function name determines the REST route exposed by the compatibility + runtime. + +The deployment platform additionally expects `/main` with a `{query: string}` +body. This platform constraint is stricter than the underlying transport. + +Each stub method returns a Future immediately. `.value()` blocks. Dispatching +and resolving inside one comprehension serializes work without raising an +error; dispatch all calls first, then resolve them. + +The workflow container also starts runtime controller code and has its own +package resolution. Probe it independently from agent images. + +## Build context and collisions + +The runtime copies project files while preserving relative paths, then writes +shared runtime modules, generated stubs, and entrypoints into the image. Later +writes can shadow project files. + +Avoid root project modules named like runtime files, including: + +```text +future.py +ventis_context.py +local_controller.py +local_controller_frontend.py +redis_client.py +grpc_options.py +bedrock.py +deploy.py +session_logging.py +workflow_launcher.py +``` + +Also avoid a yaml basename that shadows a different source module imported by an +adapter. The validator checks deterministic flat-name collisions. + +File sweep and editable-install behavior are runtime capabilities. For nested +imports, follow [packaging.md](packaging.md). + +## Dependencies and protobuf + +Agent and workflow images include a small runtime dependency set. Config +`requirements` adds source-specific distributions. A malformed requirements +value can be normalized away while image generation continues; missing imports +then surface only when the agent loads. + +The build compiles gRPC Python stubs on the host and copies them into images. +The image resolver does not necessarily know the generated-code version. A +source dependency that constrains protobuf below the host generator version can +produce a green image build that dies on: + +```text +import local_controller +``` + +Always run that probe before probing the entrypoint. Treat a generated-code / +runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter +source dependencies silently. + +## Credentials capability + +When `env_file` capability is available, the top-level config path is resolved +against the project root and passed at container start. Hidden env files are not +copied into images. Invalid paths are deploy-preflight errors. + +When the capability is unavailable, declaring `env_file` has no effect. If the +source needs credentials, report the capability blocker rather than hardcoding +or vendoring a secret. + +A source that constructs its client at import time works only when credentials +are already in the container environment. Image entrypoint probes therefore use +the same env file as deployment. + +For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). + +## Policy and provider behavior + +No policy file means unrestricted service access. If a policy exists, it needs a +non-empty rules list. Rules are evaluated by specificity and first match; +services excluded from the selected rule fail after request acceptance. + +Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior +and remote networking are covered in [ec2.md](ec2.md). + +## Cleanup boundary + +Stopping foreground deploy normally invokes controller cleanup for recorded +containers and Redis. Hard kills and failures before resource registration may +leave resources behind. + +`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and +`docker_container/`. It does not remove containers or images. Remove exact +leftovers explicitly and preserve source, port scaffolding, and requested +evidence. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md new file mode 100644 index 0000000..361acf9 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md @@ -0,0 +1,60 @@ +# Troubleshooting + +Read this after a failed build, image probe, deploy, or request. For mechanisms, +read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, +read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). + +## Build or deploy stops early + +| Symptom | Likely cause | +|---|---| +| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | +| Two services produce one image | Config names collide after lowercase normalization | +| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | +| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | +| Replica conversion `TypeError` | `replicas` is not an integer | +| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | +| Port or container name already in use | A previous deployment did not complete cleanup | + +## Container exits or serves nothing + +| Symptom | Likely cause | +|---|---| +| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | +| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | +| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | +| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | +| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | +| Third-party module is missing | Distribution is absent from source metadata and config requirements | +| Stub import raises `NameError` | yaml argument type is not a bare builtin | +| Source module behaves like an empty stub | A generated stub basename shadowed the source module | +| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | + +## Request is accepted, then fails + +| Symptom | Likely cause | +|---|---| +| Unexpected keyword argument | yaml argument name differs from adapter parameter name | +| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | +| Unauthorized service | The first matching policy rule excludes that service | +| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | +| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | +| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | +| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | +| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | + +## Deployment platform endpoint + +| Symptom | Likely cause | +|---|---| +| 404 while workflow container is healthy | Workflow function is not named `main` | +| 400 before host receives request | Body is not the platform's `{query: string}` shape | +| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | + +## Cleanup + +| Symptom | Likely cause | +|---|---| +| `ventis clean` succeeds but containers remain | The command removes generated directories only | +| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | +| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/cli/.claude/skills/porting-to-canyonos-core/validate.py b/cli/.claude/skills/porting-to-canyonos-core/validate.py new file mode 100755 index 0000000..04baf37 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/validate.py @@ -0,0 +1,1257 @@ +#!/usr/bin/env python3 +"""Preflight the runtime traps that `ventis build` cannot see. + +This deliberately does not duplicate build-time validation such as malformed +YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those +checks. This script parses Python without importing it and catches failures that +otherwise stay hidden until a container loads an agent, starts a workflow, or +serves its first request. A replica is not evidence: the controller writes +`healthy` to Redis before `_load_agent` runs. + + python validate.py [project_dir] [-c config/global_controller.yaml] + [--json] [--strict] + +Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. + +Runtime capabilities vary across CanyonOS Core installations. This script probes +the importable `ventis` package directly. A capability-gated check reports +UNAVAILABLE when its behavior cannot be proven. +""" + +import argparse +import ast +import builtins +import json +import os +import re +import sys +from typing import ClassVar + +try: + import yaml +except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency + sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") + raise SystemExit(2) from None + + +DEFAULT_CONFIG_PATH = "config/global_controller.yaml" + +# Copied flat into every image over the swept project tree, so a project module +# landing flat under one of these names is overwritten. +# ventis/stub_generator.py generate_docker / generate_workflow_docker. +RUNTIME_FLAT_NAMES = frozenset( + { + "future.py", + "ventis_context.py", + "local_controller.py", + "local_controller_frontend.py", + "redis_client.py", + "grpc_options.py", + "gpu_metrics.py", + "bedrock.py", + "deploy.py", + "session_logging.py", + "workflow_launcher.py", + } +) + +# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. +BASE_AGENT_REQUIREMENTS = [ + "grpcio", + "grpcio-tools", + "redis", + "pyyaml", + "psutil", + "ipdb", + "ipython", + "boto3", +] +# Import name -> distribution name, for the handful where they differ and the +# mismatch would otherwise be reported as a missing requirement. +IMPORT_TO_DISTRIBUTION = { + "attr": "attrs", + "bs4": "beautifulsoup4", + "cv2": "opencv-python", + "dateutil": "python-dateutil", + "dotenv": "python-dotenv", + "grpc": "grpcio", + "grpc_tools": "grpcio-tools", + "jwt": "pyjwt", + "PIL": "pillow", + "psycopg": "psycopg", + "psycopg2": "psycopg2-binary", + "pydantic_settings": "pydantic-settings", + "sklearn": "scikit-learn", + "typing_extensions": "typing-extensions", + "yaml": "pyyaml", +} + +SECRET_PATTERNS = [ + (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), + (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), + (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), + (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), +] +SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) + +ERROR = "ERROR" +WARN = "WARN" +INFO = "INFO" + + +# ------------------------------------------------------------------ # +# Capabilities # +# ------------------------------------------------------------------ # +# +# Stable labels for behavior detected from the importable runtime. They contain +# no external development metadata. + +CAPABILITY_SOURCE = { + "env_file": "runtime env-file injection", + "editable_install": "editable project installation", + "sweeps_all_files": "full project-file sweep", + "stub_two_destinations": "flat and package stub destinations", +} + + +def probe_capabilities(): + """Ask the importable ventis package what it actually supports.""" + caps = dict.fromkeys(CAPABILITY_SOURCE, False) + caps["ventis"] = False + try: + from ventis import stub_generator + except Exception: # noqa: BLE001 - a broken install must not crash the check + return caps + + caps["ventis"] = True + caps["editable_install"] = hasattr(stub_generator, "_install_step") + caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") + caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") + + import importlib + + for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): + try: + module = importlib.import_module(module_name) + except Exception: # noqa: BLE001,S112 - the other path is the live one + continue + if hasattr(module, "resolve_env_file"): + caps["env_file"] = True + break + return caps + + +# ------------------------------------------------------------------ # +# YAML with line numbers # +# ------------------------------------------------------------------ # + + +class LineDict(dict): + """A mapping that remembers where it and each of its keys were written.""" + + line = 0 + key_lines: ClassVar[dict] = {} + + +class LineLoader(yaml.SafeLoader): + pass + + +def _construct_mapping(loader, node): + data = LineDict() + yield data + data.update(loader.construct_mapping(node, deep=False)) + data.line = node.start_mark.line + 1 + data.key_lines = { + key.value: key.start_mark.line + 1 + for key, _ in node.value + if isinstance(key, yaml.ScalarNode) + } + + +LineLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping +) + + +def line_of(mapping, key=None): + """The source line of `key` inside `mapping`, or of the mapping itself.""" + if not isinstance(mapping, LineDict): + return 0 + if key is not None: + return mapping.key_lines.get(key, mapping.line) + return mapping.line + + +def load_yaml(path): + """Parse `path`, returning (data, error). Never raises.""" + try: + with open(path, "r", encoding="utf-8") as handle: + return yaml.load(handle, Loader=LineLoader), None + except Exception as exc: # noqa: BLE001 - any parse failure is a finding + return None, str(exc) + + +# ------------------------------------------------------------------ # +# Findings # +# ------------------------------------------------------------------ # + + +class Report: + def __init__(self, project_dir, capabilities): + self.project_dir = project_dir + self.capabilities = capabilities + self.findings = [] + # A peer agent is imported by the name of its generated stub, which the + # build copies flat into every image. Those are not project modules and + # need no requirement. + self.stub_module_names = set() + + def add(self, check, level, path, line, summary, mechanism): + self.findings.append( + { + "check": check, + "level": level, + "path": self.rel(path) if path else "", + "line": line or 0, + "summary": summary, + "mechanism": mechanism, + } + ) + + def error(self, check, path, line, summary, mechanism): + self.add(check, ERROR, path, line, summary, mechanism) + + def warn(self, check, path, line, summary, mechanism): + self.add(check, WARN, path, line, summary, mechanism) + + def unavailable(self, check, summary): + self.add(check, INFO, "", 0, summary, "") + + def rel(self, path): + try: + return os.path.relpath(path, self.project_dir) + except ValueError: + return path + + def counts(self): + errors = sum(1 for f in self.findings if f["level"] == ERROR) + warnings = sum(1 for f in self.findings if f["level"] == WARN) + return errors, warnings + + +# ------------------------------------------------------------------ # +# Python source helpers # +# ------------------------------------------------------------------ # + + +def parse_python(path): + """AST for `path`, or (None, error). The port is never imported.""" + try: + with open(path, "r", encoding="utf-8") as handle: + source = handle.read() + except OSError as exc: + return None, str(exc) + try: + return ast.parse(source, filename=path), None + except SyntaxError as exc: + return None, f"{exc.msg} (line {exc.lineno})" + + +def find_class(tree, name): + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == name: + return node + return None + + +def class_methods(class_node): + return { + node.name: node + for node in class_node.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + + +def parameter_names(func_node): + """Every parameter a caller can pass by keyword, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + return positional + [a.arg for a in args.kwonlyargs] + + +def required_parameters(func_node): + """Parameters with no default, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + if args.defaults: + positional = positional[: len(positional) - len(args.defaults)] + kwonly = [ + arg.arg + for arg, default in zip(args.kwonlyargs, args.kw_defaults) + if default is None + ] + return positional + kwonly + + +def toplevel_import_names(tree): + """Top-level package name of every import in the module, with line numbers.""" + names = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + names.setdefault(alias.name.split(".")[0], node.lineno) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + names.setdefault(node.module.split(".")[0], node.lineno) + return names + + +# ------------------------------------------------------------------ # +# V006-V010 adapter failures hidden by _load_agent # +# ------------------------------------------------------------------ # + +BUILTIN_TYPE_NAMES = frozenset( + name + for name in dir(builtins) + if isinstance(getattr(builtins, name), type) and not name[0].isupper() +) + + +def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): + """V006 V007 V008 V009 V010.""" + name = agent_block["name"] + functions = agent_block.get("functions") or [] + check_argument_types(report, agent_yaml_path, functions) + + entrypoint = entry.get("entrypoint") + if not entrypoint: + return + entrypoint_path = os.path.join(project_dir, entrypoint) + if not os.path.isfile(entrypoint_path): + return + + tree, error = parse_python(entrypoint_path) + if tree is None: + report.error( + "V006", + entrypoint_path, + 0, + f"the entrypoint does not parse: {error}", + "_load_agent exec_module's it and swallows the exception; the first " + "request answers 'No agent loaded'.", + ) + return + + class_node = find_class(tree, name) + if class_node is None: + classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] + found = ", ".join(classes) if classes else "no classes at all" + report.error( + "V006", + entrypoint_path, + 1, + f"no class named `{name}` at module level (found: {found})", + "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " + "the AttributeError. The class name must equal agent.name exactly.", + ) + return + + methods = class_methods(class_node) + check_constructor(report, entrypoint_path, name, methods) + + for func in functions: + if not isinstance(func, dict) or not isinstance(func.get("name"), str): + continue + check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) + + +def check_argument_types(report, agent_yaml_path, functions): + """V010 -- the type string is pasted into an ast.Name, never checked.""" + for func in functions: + if not isinstance(func, dict): + continue + for arg in func.get("arguments") or []: + if not isinstance(arg, dict) or "type" not in arg: + continue + declared = arg.get("type") + if not isinstance(declared, str): + continue # stub generation reports malformed type values + if declared in BUILTIN_TYPE_NAMES: + continue + report.error( + "V010", + agent_yaml_path, + line_of(arg, "type"), + f"`type: {declared}` is not a builtin", + "stub_generator pastes it verbatim into the generated " + "annotation, and the stub module imports only Future and " + "inspect. Anything else raises NameError when the stub is " + "imported -- after a green build. Use str int float bool dict " + "list.", + ) + + +def check_constructor(report, entrypoint_path, name, methods): + """V007 -- _load_agent calls agent_class() with no arguments.""" + init = methods.get("__init__") + if init is None: + return + required = required_parameters(init) + if required: + report.error( + "V007", + entrypoint_path, + init.lineno, + f"`{name}.__init__` requires {', '.join(required)}", + "_load_agent calls agent_class() with no arguments; the TypeError is " + "swallowed and the first request answers 'No agent loaded'. Read " + "configuration from the environment inside __init__ instead.", + ) + + +def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): + """V008 V009.""" + func_name = func["name"] + method = methods.get(func_name) + if method is None: + report.error( + "V008", + entrypoint_path, + 0, + f"`{class_name}` has no method `{func_name}`", + "The yaml declares it, so callers get a stub for it; the controller " + f"then answers \"Agent {class_name} has no method '{func_name}'\".", + ) + return + + # V009 -- nothing on the execution path awaits. + if isinstance(method, ast.AsyncFunctionDef): + report.error( + "V009", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` is `async def`", + "The executor calls method(**args) with no await, so Redis receives " + "''. Keep the signature synchronous and call " + "asyncio.run(...) inside the body.", + ) + + # V008 -- the controller calls method(**args) with the yaml's names. + declared = [ + arg["name"] + for arg in func.get("arguments") or [] + if isinstance(arg, dict) and isinstance(arg.get("name"), str) + ] + actual = parameter_names(method) + required = required_parameters(method) + + missing = [arg for arg in declared if arg not in actual] + if missing: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` has no parameter " + f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", + "LocalController does method(**args) with the yaml's argument names. " + "A mismatch is TypeError: unexpected keyword argument, at request " + f"time. See {os.path.basename(agent_yaml_path)}.", + ) + + unfilled = [arg for arg in required if arg not in declared] + if unfilled: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " + "the yaml does not declare", + "Only declared arguments are ever sent, and the generated stub gives " + "none of them a default. Declare them in the yaml or default them " + "in the signature.", + ) + + + +# ------------------------------------------------------------------ # +# V016-V018 the workflow # +# ------------------------------------------------------------------ # + + +def check_stub_imports(report, workflow_path, tree, stub_classes): + """V023 -- the workflow must import a stub as `from agents. import `. + + The build copies each stub to exactly one path, and for the workflow image + that path is agents/.py. Two ways of writing this line fail, and + the project walks you into both: the flat form is what examples/ uses, and + the class name is the one `ventis build` prints, which is not the one it + writes. + """ + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + base = alias.name.split(".")[0] + if base in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`import {alias.name}` -- the stub is at " + f"agents/{base}.py, not flat", + "The build copies a stub to one path, and for the " + "workflow that path is under agents/. This is a " + "ModuleNotFoundError the moment the workflow runs. " + f"Write `from agents.{base} import {stub_classes[base]}`.", + ) + continue + + if not isinstance(node, ast.ImportFrom) or not node.module: + continue + + module = node.module + if module in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`from {module} import ...` -- the stub is at " + f"agents/{module}.py, not flat", + "The build copies a stub to one path, and for the workflow " + "that path is under agents/. The flat form is what this " + "repository's own examples use and it raises " + "ModuleNotFoundError in the workflow image. Write " + f"`from agents.{module} import {stub_classes[module]}`.", + ) + continue + + if not module.startswith("agents."): + continue + base = module.split(".", 1)[1] + expected = stub_classes.get(base) + if expected is None: + continue + for alias in node.names: + if alias.name == expected: + continue + if alias.name == f"{expected}Stub": + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is the name the build prints, not the " + f"class it writes", + "generate_agent_stub sets class_name = agent_config['name'] " + "and then recomputes it with a 'Stub' suffix for the log " + "line only. The message names a class that does not exist; " + f"the class is `{expected}`.", + ) + else: + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is not a class the stub for {base} defines", + f"The stub's class carries the agent's own name: `{expected}`.", + ) + + +def check_workflow(report, workflow_path, stub_classes=None): + """V016 V017 V018 V023.""" + tree, error = parse_python(workflow_path) + if tree is None: + report.error("V016", workflow_path, 0, f"does not parse: {error}", "") + return + + main = None + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( + node.name == "main" + ): + main = node + break + + if main is None: + defined = [ + n.name + for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + found = ", ".join(defined) if defined else "no top-level functions" + report.error( + "V016", + workflow_path, + 1, + f"no top-level function named `main` (found: {found})", + "CanyonOS Core serves POST /, but the deployment platform's " + "test endpoint posts to a hardcoded /main. A differently named " + "workflow builds, deploys and stays unreachable -- 404, container " + "healthy.", + ) + else: + check_main_signature(report, workflow_path, main) + + if stub_classes: + check_stub_imports(report, workflow_path, tree, stub_classes) + + # V016 -- deploy() is what starts Flask. + if not any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "deploy" + for node in ast.walk(tree) + ): + report.error( + "V016", + workflow_path, + 1, + "the workflow never calls `deploy(...)`", + "workflow_launcher.py exec's this file and nothing else starts the " + "HTTP server; the container comes up serving nothing.", + ) + + check_main_guard(report, workflow_path, tree) + check_fused_fanout(report, workflow_path, tree) + + +def check_main_signature(report, workflow_path, main): + """V016 -- the platform sends exactly {"query": ...}.""" + if isinstance(main, ast.AsyncFunctionDef): + report.error( + "V016", + workflow_path, + main.lineno, + "`main` is `async def`", + "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " + "no await; the response body would be a coroutine repr.", + ) + params = parameter_names(main) + if not params: + report.error( + "V016", + workflow_path, + main.lineno, + "`main` takes no arguments", + 'The platform posts {"query": "..."} and deploy() splats the ' + "body in as kwargs -- TypeError on every request.", + ) + return + if params[0] != "query": + report.error( + "V016", + workflow_path, + main.lineno, + f"`main`'s first parameter is `{params[0]}`, not `query`", + "The platform's body schema is strictly validated as " + "{query: string}; any other key is rejected with 400 in the control " + "plane, before the request reaches the host.", + ) + extra = [p for p in required_parameters(main) if p != "query"] + if extra: + report.error( + "V016", + workflow_path, + main.lineno, + f"`main` requires {', '.join(extra)} beyond `query`", + "Only `query` is ever sent, so every other parameter needs a " + "default or the call raises on every request. Pack richer input " + "into `query`.", + ) + + +def check_main_guard(report, workflow_path, tree): + """V017 -- the workflow is exec'd, so __name__ == "__main__".""" + for node in tree.body: + if not isinstance(node, ast.If): + continue + test = node.test + if ( + isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and test.left.id == "__name__" + and any( + isinstance(c, ast.Constant) and c.value == "__main__" + for c in test.comparators + ) + ): + report.error( + "V017", + workflow_path, + node.lineno, + '`if __name__ == "__main__":` block in the workflow', + "workflow_launcher.py runs exec(open().read()), so " + "__name__ IS '__main__' here and this block executes in " + "production, at container start.", + ) + + +def check_fused_fanout(report, workflow_path, tree): + """V018 -- .value() blocks, so dispatching and resolving in one + comprehension runs the fan-out one call at a time.""" + comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) + for node in ast.walk(tree): + if not isinstance(node, comprehensions + (ast.DictComp,)): + continue + elements = ( + [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] + ) + for element in elements: + for inner in ast.walk(element): + if ( + isinstance(inner, ast.Call) + and isinstance(inner.func, ast.Attribute) + and inner.func.attr == "value" + and isinstance(inner.func.value, ast.Call) + ): + report.error( + "V018", + workflow_path, + node.lineno, + "one comprehension both dispatches a call and resolves " + "it with .value()", + ".value() blocks, so each call completes before the " + "next is dispatched. It does not error -- the fan-out " + "is just silently serial, and with it the reason to be " + "on CanyonOS Core. Dispatch every call first, then resolve: " + "futures = [a.work(i) for i in items] then " + "[f.value() for f in futures].", + ) + return + + +# ------------------------------------------------------------------ # +# V019-V020 what the copy order overwrites # +# ------------------------------------------------------------------ # + + +def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): + """V019 V020 -- later copies land on earlier ones at the context root.""" + for entry in sorted(os.listdir(project_dir)): + path = os.path.join(project_dir, entry) + if not os.path.isfile(path) or not entry.endswith(".py"): + continue + + # V019 -- the shared runtime is copied flat, after the project sweep. + if entry in RUNTIME_FLAT_NAMES: + report.error( + "V019", + path, + 1, + f"a project module named `{entry}` sits at the project root", + "The shared CanyonOS Core runtime is copied flat into the image after " + "the project sweep, so this file is overwritten by CanyonOS Core's own " + f"{entry}. Rename it or move it into a package directory.", + ) + + # V020 -- a stub is copied flat under its yaml's basename. + entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} + for yaml_path in yaml_paths: + stem = os.path.splitext(os.path.basename(yaml_path))[0] + module = f"{stem}.py" + if module in entrypoint_basenames: + continue # the entrypoint is copied last and wins its flat name back + candidate = os.path.join(project_dir, module) + if os.path.isfile(candidate): + report.error( + "V020", + candidate, + 1, + f"`{os.path.basename(yaml_path)}` generates a stub that lands on " + f"`{module}`", + "The yaml's basename names the stub, and the stub is copied flat " + "over the swept tree. Anything importing this module inside the " + "container gets the generated stub instead of the real code. " + "Rename the yaml to match its own entrypoint.", + ) + + +# ------------------------------------------------------------------ # +# V030-V031 capability-gated rules # +# ------------------------------------------------------------------ # + + +def check_env_file(report, config, config_path, project_dir): + """V030 -- gated on detected env-file injection support.""" + declared = config.get("env_file") + supported = report.capabilities.get("env_file") + + if not supported: + if declared: + report.error( + "V030", + config_path, + line_of(config, "env_file"), + f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", + "No resolve_env_file in the importable ventis package, so the " + "key is silently dropped and the container answers a provider " + "credential error on the first request. This port requires the " + "`env_file` runtime capability.", + ) + else: + report.unavailable( + "V030", + "env_file is not supported by the importable `ventis` runtime. " + "Credentials have no declared path into a container on this tree.", + ) + return + + if not declared: + report.warn( + "V030", + config_path, + line_of(config), + "no `env_file:` in the config", + "Only runtime-managed VENTIS_* variables are guaranteed without it. " + "If the source reads credentials from the environment, the first " + "request fails on a provider error.", + ) + return + + # Path existence and readability are deploy-preflight checks. Do not + # duplicate them here. + + +def check_import_root(report, project_dir, entrypoint_paths): + """V031 -- gated on detected editable-install support.""" + supported = report.capabilities.get("editable_install") + has_metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + + non_flat = [] + for path in entrypoint_paths: + tree, _ = parse_python(path) + if tree is None: + continue + for name, lineno in toplevel_import_names(tree).items(): + if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: + continue + if _resolves_flat(project_dir, name): + continue + location = _resolves_nested(project_dir, name) + if location: + non_flat.append((path, lineno, name, location)) + + if not supported: + report.unavailable( + "V031", + "the editable install (`-e .`) is not supported by the importable " + "`ventis` runtime. Only names rooted at /app import inside a container.", + ) + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, which is not at the " + "project root", + "sys.path[0] is /app and this CanyonOS Core runs no editable install, " + "so only modules swept to the root import. The adapter raises " + "ModuleNotFoundError inside _load_agent and the first request " + "answers 'No agent loaded'.", + ) + return + + if non_flat and not has_metadata: + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, and the project root " + "has no packaging metadata", + "A pyproject.toml, setup.py or setup.cfg at the port root is " + "what adds `-e .`; metadata nested inside the untouched source " + "tree is ignored. Add minimal root metadata pointing at the " + "existing package directory. Without it the install is skipped " + "silently.", + ) + + +def _resolves_flat(project_dir, name): + """Whether Python can resolve `name` with /app as its import root. + + A directory does not need __init__.py: PEP 420 namespace packages resolve + from sys.path just like regular packages. + """ + return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( + os.path.join(project_dir, name) + ) + + +def _resolves_nested(project_dir, name): + """Where below /app `name` lives but cannot resolve as a top-level name.""" + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] + if root == project_dir: + continue + if f"{name}.py" in files: + return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) + if name in dirs: + return os.path.relpath(os.path.join(root, name), project_dir) + return None + + +# ------------------------------------------------------------------ # +# W003, W006 secrets and imports a green build does not reject # +# ------------------------------------------------------------------ # + + +def check_secrets(report, port_paths): + """W003 -- env_file is the way in; nothing else is.""" + for path in port_paths: + try: + with open(path, "r", encoding="utf-8") as handle: + lines = handle.read().splitlines() + except OSError: + continue + for number, line in enumerate(lines, start=1): + for pattern, description in SECRET_PATTERNS: + if pattern.search(line): + report.warn( + "W003", + path, + number, + f"this line looks like {description}", + "Never put a secret in the source tree or the build " + "context. The build sweeps the project into every " + "image.", + ) + break + + tree, _ = parse_python(path) + if tree is None: + continue + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + if not ( + isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + and node.value.value.strip() + ): + continue + for target in node.targets: + if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): + report.warn( + "W003", + path, + node.lineno, + f"`{target.id}` is assigned a literal string", + "Read it from the environment instead; the build sweeps " + "this file into every image.", + ) + + +def _pyproject_dependencies(project_dir): + """What `-e .` installs alongside `requirements:`, or None if unreadable. + + None and the empty set mean different things here: empty means the project + declares no dependencies, None means we could not find out -- a setup.py, or + a tomllib this interpreter does not have. The caller must not treat the + second as the first, or it warns about imports the install would satisfy. + """ + path = os.path.join(project_dir, "pyproject.toml") + if not os.path.isfile(path): + return None + try: + import tomllib + except ImportError: # < 3.11 + return None + try: + with open(path, "rb") as handle: + data = tomllib.load(handle) + except Exception: # noqa: BLE001 - malformed metadata is uv's error to give + return None + deps = (data.get("project") or {}).get("dependencies") + if not isinstance(deps, list): + return None + return {_normalize_distribution(d) for d in deps if isinstance(d, str)} + + +def check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path +): + """W006 -- an import the container cannot satisfy.""" + tree, _ = parse_python(entrypoint_path) + if tree is None: + return + + declared = { + _normalize_distribution(item) + for item in (entry.get("requirements") or []) + if isinstance(item, str) + } + + # Where the editable install exists, `-e .` resolves the project's own + # [project.dependencies] in the same pass as `requirements:`. Warning about + # those is a false positive, and a false warning about a dependency is worse + # than none: it teaches the reader to dismiss this check. + editable = report.capabilities.get("editable_install") + metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + unreadable_metadata = False + if editable and metadata: + project_deps = _pyproject_dependencies(project_dir) + if project_deps is None: + unreadable_metadata = True + else: + declared |= project_deps + base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} + stdlib = getattr(sys, "stdlib_module_names", frozenset()) + + for name, lineno in sorted(toplevel_import_names(tree).items()): + if name in stdlib or name == "ventis": + continue + # Provided by the image itself: the shared runtime is copied flat, and + # every agents/*.yaml generates a stub that is copied flat too. + if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: + continue + if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): + continue + distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) + if distribution in base or distribution in declared: + continue + if unreadable_metadata: + mechanism = ( + "The container installs the base list, `requirements:`, and -- " + "since this project declares packaging metadata -- whatever " + "`-e .` resolves from it. That metadata could not be read here, " + f"so if it already requires `{name}` this line is noise; " + "otherwise it is a ModuleNotFoundError inside _load_agent and " + "'No agent loaded' on the first request." + ) + else: + mechanism = ( + "The container installs the base list plus `requirements:` and " + "nothing else, so this is a ModuleNotFoundError inside " + "_load_agent and 'No agent loaded' on the first request. If the " + f"distribution is named something other than `{name}`, declare " + f"that name in {report.rel(config_path)}." + ) + report.warn( + "W006", + entrypoint_path, + lineno, + f"`import {name}` is in neither the runtime's base list nor this " + "entry's `requirements:`", + mechanism, + ) + + +def _normalize_distribution(name): + return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( + "_", "-" + ) + + +# ------------------------------------------------------------------ # +# Driver # +# ------------------------------------------------------------------ # + + +def validate(project_dir, config_path, capabilities): + """Inspect only failures hidden behind a successful image build.""" + report = Report(project_dir, capabilities) + + # The build owns config/YAML syntax and shape validation. We read only enough + # valid structure to locate code for the deeper checks below. + config, error = load_yaml(config_path) + if error is not None or not isinstance(config, dict): + report.unavailable( + "BUILD", + "runtime preflight skipped because the config cannot be read; " + "ventis build owns and reports this error.", + ) + return report + + import glob + + yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) + report.stub_module_names = { + os.path.splitext(os.path.basename(path))[0] for path in yaml_paths + } + + agents_by_name = {} + stub_classes = {} + for path in yaml_paths: + data, yaml_error = load_yaml(path) + agent = data.get("agent") if isinstance(data, dict) else None + name = agent.get("name") if isinstance(agent, dict) else None + if yaml_error is not None or not isinstance(name, str): + continue # ventis build reports malformed agent declarations + agents_by_name[name] = (path, agent) + stub_classes[os.path.splitext(os.path.basename(path))[0]] = name + + entries = config.get("agents") + if not isinstance(entries, list): + report.unavailable( + "BUILD", + "runtime preflight skipped because `agents:` is not a list; " + "ventis build owns and reports this error.", + ) + return report + + entrypoints = [] + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": + continue + name = entry.get("name") + entrypoint = entry.get("entrypoint") + if isinstance(entrypoint, str): + entrypoints.append(entrypoint) + if name in agents_by_name: + yaml_path, agent_block = agents_by_name[name] + check_adapter(report, yaml_path, agent_block, entry, project_dir) + entrypoint_path = os.path.join(project_dir, entrypoint or "") + if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): + check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path + ) + + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": + continue + workflow_file = entry.get("workflow_file") + if not isinstance(workflow_file, str): + continue + workflow_path = os.path.join(project_dir, workflow_file) + if os.path.isfile(workflow_path): + check_workflow(report, workflow_path, stub_classes) + + # These survive a green build and otherwise surface only in a container or + # on its first request. + check_flat_collisions(report, project_dir, yaml_paths, entrypoints) + check_env_file(report, config, config_path, project_dir) + + entrypoint_paths = [ + os.path.join(project_dir, e) + for e in entrypoints + if os.path.isfile(os.path.join(project_dir, e)) + ] + check_import_root(report, project_dir, entrypoint_paths) + + port_paths = list(entrypoint_paths) + for entry in entries: + if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): + candidate = os.path.join(project_dir, entry["workflow_file"]) + if os.path.isfile(candidate): + port_paths.append(candidate) + + # Secret detection remains because a green image build would permanently + # bake the credential into every image. + check_secrets(report, port_paths) + return report + + + +# ------------------------------------------------------------------ # +# Output # +# ------------------------------------------------------------------ # + +LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} + + +def _wrap(text, width, indent): + words = text.split() + lines = [] + current = "" + for word in words: + candidate = f"{current} {word}".strip() + if len(candidate) + len(indent) > width and current: + lines.append(indent + current) + current = word + else: + current = candidate + if current: + lines.append(indent + current) + return lines + + +def print_report(report, project_dir): + caps = report.capabilities + if not caps.get("ventis"): + print("ventis is not importable here -- capability-gated rules are") + print("reported UNAVAILABLE rather than checked.\n") + else: + print("CanyonOS Core capabilities detected:") + for key, source in CAPABILITY_SOURCE.items(): + mark = "yes" if caps.get(key) else "no " + print(f" {mark} {key:<22} {source}") + print() + + findings = sorted( + report.findings, + key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), + ) + for finding in findings: + where = finding["path"] + if where and finding["line"]: + where = f"{where}:{finding['line']}" + header = f"{finding['check']} {finding['level']:<5}" + print(f"{header} {where}" if where else header) + for line in _wrap(finding["summary"], 78, " "): + print(line) + if finding["mechanism"]: + for line in _wrap(finding["mechanism"], 78, " "): + print(line) + print() + + errors, warnings = report.counts() + if not findings: + print(f"{project_dir}: clean.") + return + print(f"{errors} error(s), {warnings} warning(s).") + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Check a CanyonOS Core port against the rules in SKILL.md." + ) + parser.add_argument( + "project_dir", nargs="?", default=".", help="the port's project root" + ) + parser.add_argument( + "-c", + "--config", + default=DEFAULT_CONFIG_PATH, + help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", + ) + parser.add_argument("--json", action="store_true", help="emit findings as JSON") + parser.add_argument( + "--strict", action="store_true", help="fail on warnings as well as errors" + ) + args = parser.parse_args(argv) + + project_dir = os.path.abspath(args.project_dir) + config_path = ( + args.config + if os.path.isabs(args.config) + else os.path.join(project_dir, args.config) + ) + + capabilities = probe_capabilities() + report = validate(project_dir, config_path, capabilities) + errors, warnings = report.counts() + + if args.json: + print( + json.dumps( + { + "project_dir": project_dir, + "capabilities": capabilities, + "errors": errors, + "warnings": warnings, + "findings": report.findings, + }, + indent=2, + ) + ) + else: + print_report(report, report.rel(project_dir) or project_dir) + + if errors or (args.strict and warnings): + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..de66cd0 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,21 @@ +Lightweight CLI for CanyonOS + +Serves as a thin API layer, connecting to the global controller container. + +## Serve + +`canyonos serve -c config/global_controller.yaml` starts the local CanyonOS dashboard. It reads +`database.url` from the config and writes only `CANYONOS_`-prefixed settings to the current `.env`, +leaving other lines unchanged. + + +### To Republish to PyPi + +```Terminal +cd cli +# Go into, pyproject.toml, and increment version number +rm -rf dist/ # Removes the old distro, causes conflicts + +uv build +uv publish # Needs PyPi Auth Token, ask Saaketh +``` \ No newline at end of file diff --git a/cli/canyonos/__init__.py b/cli/canyonos/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/canyonos/clean.py b/cli/canyonos/clean.py new file mode 100644 index 0000000..aabf105 --- /dev/null +++ b/cli/canyonos/clean.py @@ -0,0 +1,28 @@ +""" +Remove generated stubs, gRPC files, and Docker build contexts. + +Ported directly over from canyonos, moving the logic into here. +""" + +import os +import shutil + + +def run_clean(): + project_dir = os.getcwd() + + paths_to_clean = [ + os.path.join(project_dir, "stubs"), + os.path.join(project_dir, "grpc_stubs"), + os.path.join(project_dir, "docker_container"), + ] + + for path in paths_to_clean: + if os.path.exists(path): + print(f"Cleaning {path}...") + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + print("Clean complete.") diff --git a/cli/canyonos/config.py b/cli/canyonos/config.py new file mode 100644 index 0000000..226312e --- /dev/null +++ b/cli/canyonos/config.py @@ -0,0 +1,345 @@ +""" +Logic for `canyonos config`: view or change project/deploy configuration. +""" + +import os + +import yaml +from rich.console import Console +from rich.table import Table +from ruamel.yaml import YAML + +from canyonos.constants import default_config_path +from canyonos.theme import GREEN, WHITE +from utils.tui import DELETE_ACTION, QUIT_ACTION, select_menu + +BACK = "__back__" + +OPTIONS = [ + ("view", "View"), + ("change", "Change"), +] + +BORDER = GREEN +HEADER = f"bold {GREEN}" + +# Rendered as their own tables (in this order); everything else scalar at the +# top level is collected into a single "General" table. +STRUCTURED_KEYS = ("agents", "otel") + + +def _fmt(value): + """Render a YAML value as a compact, single-cell string.""" + if value is None: + return "-" + if isinstance(value, bool): + return "yes" if value else "no" + if isinstance(value, list): + return ", ".join(_fmt(v) for v in value) if value else "-" + if isinstance(value, dict): + return ", ".join(f"{k}={_fmt(v)}" for k, v in value.items()) if value else "-" + return str(value) + + +def _agents_table(agents): + table = Table(title="Agents", border_style=BORDER, header_style=HEADER, title_style=HEADER) + for col in ("Name", "Type", "Replicas", "CPU", "Mem", "Provider", "Port", "Entrypoint"): + table.add_column(col) + + for agent in agents: + resources = agent.get("resources") or {} + # Workflows carry `workflow_file` + `api_port`; plain agents carry + # `entrypoint` + `redis_port`. + entry = agent.get("entrypoint") or agent.get("workflow_file") or "-" + port = agent.get("api_port") or agent.get("redis_port") + table.add_row( + _fmt(agent.get("name")), + agent.get("type", "agent"), + _fmt(agent.get("replicas")), + _fmt(resources.get("cpu")), + _fmt(resources.get("memory")), + _fmt(agent.get("provider")), + _fmt(port), + entry, + ) + return table + + +def _otel_table(otel): + destinations = (otel or {}).get("destinations") or [] + table = Table( + title="OTel Destinations", border_style=BORDER, header_style=HEADER, title_style=HEADER + ) + for col in ("Name", "Protocol", "Endpoint", "Insecure", "Headers"): + table.add_column(col) + + for dest in destinations: + headers = dest.get("headers") or {} + table.add_row( + _fmt(dest.get("name")), + _fmt(dest.get("protocol")), + _fmt(dest.get("endpoint")), + _fmt(dest.get("insecure", False)), + ", ".join(headers.keys()) if headers else "-", + ) + return table + + +def _kv_table(title, data): + """A two-column Setting/Value table from a flat-ish dict (or single value).""" + table = Table(title=title, border_style=BORDER, header_style=HEADER, title_style=HEADER) + table.add_column("Setting", style="bold") + table.add_column("Value") + + if isinstance(data, dict): + for key, value in data.items(): + table.add_row(str(key), _fmt(value)) + else: + table.add_row(title, _fmt(data)) + return table + + +def run_view_config(config_path=None): + config_path = config_path or default_config_path() + console = Console() + + if not os.path.isfile(config_path): + console.print(f"[red]Config file not found: {config_path}[/red]") + return + + with open(config_path) as f: + config = yaml.safe_load(f) or {} + + console.print(_agents_table(config.get("agents") or [])) + console.print() + + if config.get("otel"): + console.print(_otel_table(config["otel"])) + console.print() + + # Every other top-level key: dicts get their own table, bare scalars are + # gathered into a single "General" table. + general = {} + for key, value in config.items(): + if key in STRUCTURED_KEYS: + continue + if isinstance(value, dict): + console.print(_kv_table(key, value)) + console.print() + else: + general[key] = value + + if general: + console.print(_kv_table("General", general)) + + +def _is_leaf(value): + """A value the user edits directly: any scalar, or a list of only scalars. + + Lists of mappings (agents, otel.destinations) are containers to drill into; + lists of plain scalars (requirements, security_group_ids) are edited whole + via comma-separated input. + """ + if isinstance(value, dict): + return False + if isinstance(value, list): + return all(not isinstance(item, (dict, list)) for item in value) + return True + + +def _preview(value): + if isinstance(value, dict): + return f"{{{len(value)} keys}}" + if isinstance(value, list) and not _is_leaf(value): + return f"[{len(value)} items]" + return _fmt(value) + + +def _seq_label(index, item): + if isinstance(item, dict) and item.get("name"): + return str(item["name"]) + return f"[{index}]" + + +def _cast(raw, current): + """Coerce the typed string to the current value's type. Raises ValueError.""" + # bool must precede int: bool is a subclass of int. + if isinstance(current, bool): + low = raw.strip().lower() + if low in ("true", "yes", "y", "1"): + return True + if low in ("false", "no", "n", "0"): + return False + raise ValueError("expected yes/no") + if isinstance(current, int): + return int(raw) + if isinstance(current, float): + return float(raw) + if isinstance(current, list): + return [part.strip() for part in raw.split(",") if part.strip()] + return raw + + +class _Screen: + """Owns the alt-screen: clears and redraws a persistent breadcrumb header + (plus a transient status line) before each menu/prompt, so the change + session replaces the view in place instead of scrolling. + """ + + def __init__(self, console): + self.console = console + self.status = "" + + def render(self, breadcrumb): + self.console.clear() + path = " \u203a ".join(str(part) for part in breadcrumb) if breadcrumb else "config" + self.console.print(f"[bold {GREEN}]CanyonOS[/] [{WHITE}]config[/]") + self.console.print(f"[{WHITE}]{path}[/]") + if self.status: + self.console.print(f"[{GREEN}]{self.status}[/]") + self.console.print() + + +def _edit_leaf(screen, parent, key, breadcrumb): + """Prompt for and apply a new value for parent[key]. Returns True if changed.""" + screen.render(breadcrumb) + console = screen.console + current = parent[key] + label = key if not isinstance(key, int) else f"item {key}" + console.print(f"[bold]{label}[/bold] current: {_fmt(current)}") + if isinstance(current, list): + console.print("[dim]enter comma-separated values[/dim]") + + raw = input("New value (blank to cancel): ").strip() + if raw == "": + return False + + try: + parent[key] = _cast(raw, current) + except ValueError as exc: + screen.status = f"Invalid value: {exc}" + return False + + screen.status = f"Set {label} = {_fmt(parent[key])}" + return True + + +def _confirm_delete(screen, node, key, breadcrumb): + """Yes/No confirm menu for deleting node[key]. Returns True to delete.""" + screen.render(breadcrumb) + label = key if isinstance(node, dict) else _seq_label(key, node[key]) + options = [("yes", f"Yes, delete '{label}'"), ("no", "No, keep it")] + choice = select_menu( + options, + title=f"Delete '{label}' ({_preview(node[key])}) and everything inside?", + ) + return choice == "yes" + + +def _navigate(screen, node, breadcrumb): + """Drill into a mapping/sequence. Returns True if any value was changed or + deleted, None if the user backed out of this level, or QUIT_ACTION if the + user quit (which unwinds the whole session from any depth).""" + while True: + screen.render(breadcrumb) + if isinstance(node, dict): + options = [(k, f"{k}: {_preview(v)}") for k, v in node.items()] + else: # list + options = [(i, f"{_seq_label(i, item)}: {_preview(item)}") for i, item in enumerate(node)] + options.append((BACK, "\u2190 Back")) + + choice = select_menu( + options, title="Select a field (d to delete)", deletable=True, quittable=True + ) + if choice is None: + return None + # 'q' anywhere -> unwind the entire session, not just this level. + if choice is QUIT_ACTION: + return QUIT_ACTION + + # 'd' over an item -> (DELETE_ACTION, hovered_value). + if isinstance(choice, tuple) and choice[0] is DELETE_ACTION: + target = choice[1] + if target == BACK: + continue # the Back entry isn't deletable + if _confirm_delete(screen, node, target, breadcrumb): + label = target if isinstance(node, dict) else _seq_label(target, node[target]) + del node[target] + screen.status = f"Deleted '{label}'" + return True + continue # delete cancelled: stay on this menu + + if choice == BACK: + return None + + child = node[choice] + label = choice if isinstance(node, dict) else _seq_label(choice, child) + if _is_leaf(child): + if _edit_leaf(screen, node, choice, breadcrumb + [str(label)]): + return True + # cancelled/invalid: stay on this menu + else: + result = _navigate(screen, child, breadcrumb + [str(label)]) + if result is QUIT_ACTION: + return QUIT_ACTION + if result: + return True + # backed out of the child: stay on this menu + + +def run_change_config(config_path=None): + config_path = config_path or default_config_path() + console = Console() + + if not os.path.isfile(config_path): + console.print(f"[red]Config file not found: {config_path}[/red]") + return + + # Round-trip loader preserves comments, key order, quoting and ${ENV} refs. + yaml_rt = YAML() + yaml_rt.preserve_quotes = True + # Match the project's YAML style so edits don't reflow list indentation: + # block sequences indented under their key (` - item`). + yaml_rt.indent(mapping=2, sequence=4, offset=2) + with open(config_path) as f: + data = yaml_rt.load(f) + + if not data: + console.print("[yellow]Config is empty; nothing to change.[/yellow]") + return + + screen = _Screen(console) + saves = 0 + # Alternate screen: the whole session replaces the view, and the terminal + # scrollback is restored untouched on exit. + console.set_alt_screen(True) + try: + while True: + changed = _navigate(screen, data, ["config"]) + # None = backed out at root, QUIT_ACTION = quit from any depth. + if changed is None or changed is QUIT_ACTION: + break + with open(config_path, "w") as f: + yaml_rt.dump(data, f) + saves += 1 + screen.status = f"Saved to {config_path}" + finally: + console.set_alt_screen(False) + + if saves: + console.print(f"[{GREEN}]Saved {saves} change(s) to {config_path}[/]") + else: + console.print("No changes made.") + + +def run_config(): + console = Console() + choice = select_menu(OPTIONS, title="What do you want to do?") + if choice is None: + console.print("Cancelled.") + return + + if choice == "view": + run_view_config() + elif choice == "change": + run_change_config() diff --git a/cli/canyonos/constants.py b/cli/canyonos/constants.py new file mode 100644 index 0000000..d34316e --- /dev/null +++ b/cli/canyonos/constants.py @@ -0,0 +1,9 @@ +"""Shared constants for the canyonos CLI.""" + +import os + + +def default_config_path(): + """Global controller config for the current directory, preferring the .car artifact layout.""" + car = os.path.join(".car", "config", "global_controller.yaml") + return car if os.path.isfile(car) else os.path.join("config", "global_controller.yaml") diff --git a/cli/canyonos/dashboard.compose.yml b/cli/canyonos/dashboard.compose.yml new file mode 100644 index 0000000..db775aa --- /dev/null +++ b/cli/canyonos/dashboard.compose.yml @@ -0,0 +1,42 @@ +services: + # Fast-path bundled DB, hardcoded creds -- fine for local dev, not for anything real. + db: + image: postgres:16-alpine + environment: + POSTGRES_USER: canyonos + POSTGRES_PASSWORD: canyonos + POSTGRES_DB: canyonos + healthcheck: + test: ["CMD-SHELL", "pg_isready -U canyonos"] + interval: 2s + timeout: 3s + retries: 20 + ports: + - "127.0.0.1:5432:5432" + + api: + image: ${CANYONOS_API_IMAGE} + depends_on: + db: + condition: service_healthy + # Published so a GC container can POST OTLP spans to /v1/traces via + # host.docker.internal; that route also renames ventis' `project_id` + # attribute to the `canyon.project.id` every dashboard query filters on. + ports: + - "127.0.0.1:3000:3000" + environment: + DATABASE_URL: postgresql://canyonos:canyonos@db:5432/canyonos + JWT_SECRET: ${CANYONOS_JWT_SECRET} + CANYONOS_DISABLE_AUTH: "true" + CANYONOS_REDIS_HOST: ${CANYONOS_REDIS_HOST} + CANYONOS_REDIS_PORT: ${CANYONOS_REDIS_PORT} + LOG_LEVEL: info + extra_hosts: + - host.docker.internal:host-gateway + web: + image: ${CANYONOS_WEB_IMAGE} + depends_on: + api: + condition: service_healthy + ports: + - "127.0.0.1:${CANYONOS_WEB_PORT}:8080" diff --git a/cli/canyonos/dashboard_stack.py b/cli/canyonos/dashboard_stack.py new file mode 100644 index 0000000..6abcfb1 --- /dev/null +++ b/cli/canyonos/dashboard_stack.py @@ -0,0 +1,518 @@ +"""Manage the local CanyonOS dashboard stack.""" + +from __future__ import annotations + +import importlib.resources +import json +import os +import re +import secrets +import shutil +import socket +import subprocess +import tempfile +import time +import urllib.error +import urllib.request +from contextlib import ExitStack +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable +from urllib.parse import urlsplit, urlunsplit + +import yaml + +from canyonos.constants import default_config_path + +COMPOSE_PROJECT = "canyonos-dashboard" +STACK_VERSION = "v0.1.0-rc.2" +API_IMAGE = f"ghcr.io/canyoncodecoreai/canyonos-api:{STACK_VERSION}" +WEB_IMAGE = f"ghcr.io/canyoncodecoreai/canyonos-web:{STACK_VERSION}" +HOST_GATEWAY = "host.docker.internal" +REDIS_HOST = HOST_GATEWAY +REDIS_PORT = "6379" +ENV_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +@dataclass(frozen=True) +class ServeResult: + ok: bool + phase: str + message: str + url: str | None = None + log_path: str | None = None + + +class PhaseFailure(Exception): + def __init__(self, phase: str, message: str, *, had_containers: bool | None = None): + super().__init__(message) + self.phase = phase + self.message = message + self.had_containers = had_containers + + +@dataclass(frozen=True) +class DashboardStack: + database_url: str | None + state_dir: Path + project_dir: Path + web_port: int = 8080 + + @property + def env_path(self) -> Path: + return self.project_dir / ".env" + + +def _run(argv: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(argv, capture_output=True, check=False, text=True) + + +def _state_dir() -> Path: + return Path.home() / ".canyonos" / "dashboard" + + +def _compose_argv(stack: DashboardStack, manifest: Path) -> list[str]: + # Absolute, not "./.env": `canyonos serve` may cd into .car/ before + # running, so a cwd-relative path would miss the project root .env that + # `prepare()` actually writes to (stack.env_path). + return [ + "docker", + "compose", + "-p", + COMPOSE_PROJECT, + "--env-file", + str(stack.env_path), + "-f", + str(manifest), + ] + + +def _managed_database_url(database_url: str) -> tuple[str, str | None]: + parsed = urlsplit(database_url) + if parsed.hostname not in {"localhost", "127.0.0.1"}: + return database_url, None + + hostname = parsed.hostname + credentials = "" + if parsed.username is not None: + credentials = parsed.username + if parsed.password is not None: + credentials = f"{credentials}:{parsed.password}" + credentials = f"{credentials}@" + port = f":{parsed.port}" if parsed.port is not None else "" + rewritten = urlunsplit( + (parsed.scheme, f"{credentials}{HOST_GATEWAY}{port}", parsed.path, parsed.query, parsed.fragment) + ) + return rewritten, hostname + + +def _existing_dashboard_port() -> int | None: + """The host port an already-running dashboard `web` container owns, if any + -- so re-running `canyonos serve` reconnects to the same stack instead of + picking a new port out from under it.""" + try: + result = _run(["docker", "container", "inspect", "canyonos-dashboard-web-1"]) + except OSError: + return None + if result.returncode != 0: + return None + + try: + containers = json.loads(result.stdout) + bindings = containers[0]["NetworkSettings"]["Ports"].get("8080/tcp") or [] + except (IndexError, KeyError, TypeError, json.JSONDecodeError): + return None + + for binding in bindings: + if binding.get("HostIp") in {"127.0.0.1", "0.0.0.0", "::"}: + try: + return int(binding["HostPort"]) + except (KeyError, TypeError, ValueError): + continue + return None + + +def _port_is_free(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + try: + probe.bind(("127.0.0.1", port)) + except OSError: + return False + return True + + +def _find_web_port(start: int = 8080, max_attempts: int = 50) -> int: + """First free port at or after `start` -- same retry-on-conflict shape as + init.py's GC port selection, so an unrelated process/container squatting + on 8080 (e.g. a deployed Workflow's own api_port) doesn't hard-block serve. + """ + for port in range(start, start + max_attempts): + if _port_is_free(port): + return port + raise PhaseFailure( + "validate", f"no free port found for the dashboard after {max_attempts} attempts starting at {start}" + ) + + +def _load_project_config(config_path: str) -> tuple[object, Path]: + project_root = Path(os.path.abspath(os.path.join(os.path.dirname(config_path), ".."))) + # `canyonos serve` cds into .car/ before calling here, so the naive + # parent-of-parent lands on .car itself -- go up one more level to reach + # the actual project root, where .env lives. + if project_root.name == ".car": + project_root = project_root.parent + dotenv_path = project_root / ".env" + if dotenv_path.is_file(): + with dotenv_path.open(encoding="utf-8") as dotenv: + for line in dotenv: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = _env_value(value) + if key and key not in os.environ: + os.environ[key] = value + + with open(config_path, encoding="utf-8") as config_file: + config = yaml.safe_load(config_file) + return _expand_env_value(config), project_root + + +def _expand_env_value(value: object) -> object: + if isinstance(value, str): + return ENV_REFERENCE.sub(lambda match: os.environ.get(match.group(1), match.group(0)), value) + if isinstance(value, dict): + return {key: _expand_env_value(item) for key, item in value.items()} + if isinstance(value, list): + return [_expand_env_value(item) for item in value] + return value + + +def validate(config_path: str) -> DashboardStack: + if shutil.which("docker") is None: + raise PhaseFailure("validate", "docker is not on PATH") + + try: + if _run(["docker", "info"]).returncode != 0: + raise PhaseFailure("validate", "docker daemon or socket is unavailable") + if _run(["docker", "compose", "version"]).returncode != 0: + raise PhaseFailure("validate", "docker compose is unavailable") + except OSError: + raise PhaseFailure("validate", "docker daemon or socket is unavailable") + + try: + # Keep interpolation consistent with GlobalController._load_config in ventis/controller/global_controller.py. + config, project_root = _load_project_config(config_path) + except (OSError, yaml.YAMLError): + raise PhaseFailure("validate", f"config file is not readable: {config_path}") + + # database.url is optional -- the dashboard works without a database configured + # (e.g. OTLP-only setups); if present, it still needs to actually be usable. + database = config.get("database") if isinstance(config, dict) else None + database_url = database.get("url") if isinstance(database, dict) else None + if database_url is not None: + if not isinstance(database_url, str) or not database_url.strip(): + raise PhaseFailure("validate", "database.url must be a non-empty string") + unresolved = ENV_REFERENCE.search(database_url) + if unresolved: + name = unresolved.group(1) + raise PhaseFailure( + "validate", f"database.url needs ${{{name}}}, which is not set in the project .env" + ) + database_url = database_url.strip() + + state_dir = _state_dir() + try: + state_dir.mkdir(parents=True, exist_ok=True) + probe_path = state_dir / ".write-probe" + with open(probe_path, "w", encoding="utf-8") as probe: + probe.write("") + probe_path.unlink() + except OSError: + raise PhaseFailure("validate", "dashboard state directory is not writable") + + web_port = _existing_dashboard_port() or _find_web_port() + + return DashboardStack(database_url, state_dir, project_root, web_port) + + +def _env_value(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + return value[1:-1] + return value + + +def _read_existing_secret(env_path: Path) -> str | None: + try: + lines = env_path.read_text(encoding="utf-8").splitlines() + except OSError: + return None + + for line in lines: + key, separator, value = line.partition("=") + if separator and key == "CANYONOS_JWT_SECRET" and _env_value(value): + return _env_value(value) + return None + + +def _write_private_file(path: Path, contents: str) -> None: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(contents) + + +def _env_line(key: str, value: str) -> str: + if " " in value or "#" in value: + return f'{key}="{value}"\n' + return f"{key}={value}\n" + + +def _write_project_env(env_path: Path, managed_env: dict[str, str]) -> None: + try: + lines = env_path.read_text(encoding="utf-8").splitlines(keepends=True) + except FileNotFoundError: + lines = [] + + managed_keys = set(managed_env) + replaced: set[str] = set() + updated_lines: list[str] = [] + for line in lines: + key, separator, _ = line.partition("=") + if separator and key in managed_keys: + if key not in replaced: + updated_lines.append(_env_line(key, managed_env[key])) + replaced.add(key) + continue + updated_lines.append(line) + + for key, value in managed_env.items(): + if key not in replaced: + updated_lines.append(_env_line(key, value)) + + descriptor, temporary_name = tempfile.mkstemp(prefix=".env.", dir=env_path.parent) + temporary_path = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + os.fchmod(output.fileno(), 0o600) + output.writelines(updated_lines) + os.replace(temporary_path, env_path) + except Exception: + try: + temporary_path.unlink() + except FileNotFoundError: + pass + raise + + +def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: + try: + stack.state_dir.mkdir(parents=True, exist_ok=True) + os.chmod(stack.state_dir, 0o700) + managed_env = { + "CANYONOS_JWT_SECRET": _read_existing_secret(stack.env_path) or secrets.token_urlsafe(32), + "CANYONOS_REDIS_HOST": REDIS_HOST, + "CANYONOS_REDIS_PORT": REDIS_PORT, + "CANYONOS_API_IMAGE": API_IMAGE, + "CANYONOS_WEB_IMAGE": WEB_IMAGE, + "CANYONOS_WEB_PORT": str(stack.web_port), + } + rewritten_host = None + if stack.database_url is not None: + managed_database_url, rewritten_host = _managed_database_url(stack.database_url) + managed_env["CANYONOS_DATABASE_URL"] = managed_database_url + _write_project_env(stack.env_path, managed_env) + (stack.state_dir / "stack.json").write_text( + json.dumps( + { + "schema_version": 1, + "stack_version": STACK_VERSION, + "compose_project": COMPOSE_PROJECT, + } + ) + + "\n", + encoding="utf-8", + ) + except (OSError, ValueError): + raise PhaseFailure("prepare", "could not prepare the dashboard state directory") + + message = "dashboard state prepared" + if rewritten_host: + message = ( + f"database host {rewritten_host} is reachable from the stack as host.docker.internal" + ) + return managed_env, message + + +def _redact_stack_text(text: str, stack: DashboardStack, managed_env: dict[str, str]) -> str: + secret = managed_env["CANYONOS_JWT_SECRET"] + redacted = redact_logs(text, secret) + if stack.database_url is not None: + redacted = redact_logs(redacted.replace(stack.database_url, "[redacted]"), secret) + return redacted + + +def _last_stderr_line(result: subprocess.CompletedProcess[str]) -> str | None: + return next((line.strip() for line in reversed(result.stderr.splitlines()) if line.strip()), None) + + +def _command_failure_message( + message: str, + result: subprocess.CompletedProcess[str], + stack: DashboardStack, + managed_env: dict[str, str], +) -> str: + detail = _last_stderr_line(result) + if detail is None: + return message + return f"{message}: {_redact_stack_text(detail, stack, managed_env)}" + + +def pull( + stack: DashboardStack, + manifest: Path, + managed_env: dict[str, str], + had_containers: bool, +) -> str: + try: + result = _run([*_compose_argv(stack, manifest), "pull"]) + except OSError: + raise PhaseFailure("pull", "could not run docker compose pull", had_containers=had_containers) + if result.returncode != 0: + raise PhaseFailure( + "pull", + _command_failure_message("docker compose pull failed", result, stack, managed_env), + had_containers=had_containers, + ) + return "dashboard images pulled" + + +def _project_has_running_containers(stack: DashboardStack, manifest: Path) -> bool: + try: + result = _run([*_compose_argv(stack, manifest), "ps", "-q"]) + except OSError: + return False + return result.returncode == 0 and bool(result.stdout.strip()) + + +def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> bool: + had_containers = _project_has_running_containers(stack, manifest) + # The api reads the controller's Redis identity once at startup to create + # its project row, so a surviving container keeps serving whichever project + # was deployed before it. Replace it every serve rather than reuse it. + _run([*_compose_argv(stack, manifest), "rm", "-sf", "api"]) + try: + result = _run( + [*_compose_argv(stack, manifest), "up", "-d", "--wait", "--wait-timeout", "180"] + ) + except OSError: + raise PhaseFailure("start", "could not run docker compose up", had_containers=had_containers) + if result.returncode != 0: + raise PhaseFailure( + "start", + _command_failure_message("docker compose up failed", result, stack, managed_env), + had_containers=had_containers, + ) + return had_containers + + +def verify(port: int) -> str: + dashboard_url = f"http://127.0.0.1:{port}" + deadline = time.monotonic() + 30 + endpoints = (f"{dashboard_url}/healthz", f"{dashboard_url}/api/healthz") + while time.monotonic() < deadline: + healthy = True + for endpoint in endpoints: + try: + response = urllib.request.urlopen(endpoint, timeout=5) + try: + status = response.status + finally: + response.close() + except (OSError, urllib.error.URLError): + healthy = False + break + if status != 200: + healthy = False + break + if healthy: + return dashboard_url + if time.monotonic() < deadline: + time.sleep(1) + raise PhaseFailure("verify", "dashboard health checks did not return 200 within 30 seconds") + + +def redact_logs(logs: str, jwt_secret: str) -> str: + redacted = logs.replace(jwt_secret, "[redacted]") + return re.sub(r"://[^/\s@]+@", "://[redacted]@", redacted) + + +def _capture_failure_logs(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> Path: + try: + result = _run([*_compose_argv(stack, manifest), "logs", "--no-color", "--tail", "200"]) + logs = f"{result.stdout}\n{result.stderr}" + except OSError: + logs = "Unable to collect docker compose logs." + + log_dir = stack.state_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + log_path = log_dir / f"serve-{timestamp}.log" + _write_private_file( + log_path, + _redact_stack_text(logs, stack, managed_env), + ) + return log_path + + +def _cleanup(stack: DashboardStack, manifest: Path) -> None: + try: + _run([*_compose_argv(stack, manifest), "down"]) + except OSError: + return + + +def run_dashboard( + config_path: str | None = None, + phase_reporter: Callable[[str, str], None] | None = None, +) -> ServeResult: + config_path = config_path or default_config_path() + def report(result: ServeResult) -> None: + if phase_reporter is not None: + phase_reporter(result.phase, result.message) + + stack: DashboardStack | None = None + managed_env: dict[str, str] | None = None + manifest: Path | None = None + had_containers = False + with ExitStack() as resources: + try: + stack = validate(config_path) + report(ServeResult(True, "validate", "dashboard prerequisites validated")) + + managed_env, prepare_message = prepare(stack) + report(ServeResult(True, "prepare", prepare_message)) + + manifest_resource = importlib.resources.files("canyonos").joinpath("dashboard.compose.yml") + manifest = resources.enter_context(importlib.resources.as_file(manifest_resource)) + had_containers_before_pull = _project_has_running_containers(stack, manifest) + pull_message = pull(stack, manifest, managed_env, had_containers_before_pull) + report(ServeResult(True, "pull", pull_message)) + + had_containers = start(stack, manifest, managed_env) + report(ServeResult(True, "start", "dashboard stack started")) + + url = verify(stack.web_port) + report(ServeResult(True, "verify", "dashboard health checks passed", url)) + return ServeResult(True, "verify", "dashboard health checks passed", url) + except PhaseFailure as failure: + log_path = None + if failure.phase in {"pull", "start", "verify"} and stack and managed_env and manifest: + log_path = _capture_failure_logs(stack, manifest, managed_env) + if not (failure.had_containers if failure.had_containers is not None else had_containers): + _cleanup(stack, manifest) + return ServeResult( + False, failure.phase, failure.message, None, str(log_path) if log_path else None + ) diff --git a/cli/canyonos/deploy.py b/cli/canyonos/deploy.py new file mode 100644 index 0000000..a2239a2 --- /dev/null +++ b/cli/canyonos/deploy.py @@ -0,0 +1,86 @@ +""" +Logic for `canyonos deploy`: copy the project into the container's /workspace +volume (via `canyonos sync`), then tell the Global Controller container to +build and deploy it. The container's `ventis deploy` handles both the build +(stubs, protos, Docker images) and the launch -- the CLI just ships files, +triggers it, and streams the logs. + +Once the deploy's logs report the workflow is actually up, `canyonos serve` +is kicked off automatically so the local dashboard is ready without an extra +manual step. +""" + +import json +import subprocess +import urllib.error +import urllib.request + +from canyonos.constants import default_config_path +from canyonos.init import load_state, run_init +from canyonos.serve import run_serve +from canyonos.sync import run_sync + +# Logged exactly once by GlobalController.run(), right after `_wait_for_healthy()` +# returns -- the signal that the workflow finished coming up and entered its +# steady-state polling loop. +_WORKFLOW_UP_MARKER = "Global controller started, polling every" + + +def run_deploy(config_path=None, serve=True): + config_path = config_path or default_config_path() + run_init() + + # Copy the current project into the container before building/deploying. + if not run_sync(): + return + + state = load_state() + + url = f"http://127.0.0.1:{state['port']}/deploy" + body = json.dumps({"config_path": config_path}).encode() + req = urllib.request.Request( + url, data=body, headers={"Content-Type": "application/json"}, method="POST" + ) + + try: + with urllib.request.urlopen(req) as resp: + json.loads(resp.read()) + _stream_logs_and_autoserve(state["container_id"], serve=serve) + except urllib.error.HTTPError as e: + data = json.loads(e.read()) + print(f"Deploy failed: {data.get('error')}") + except urllib.error.URLError as e: + print(f"Could not reach Global Controller container: {e}") + + +def _stream_logs_and_autoserve(container_id, serve=True): + """Tail the GC container's logs (same as before), and -- unless disabled + via `serve=False` -- launch `canyonos serve` the moment they show the + workflow is up, so the dashboard is ready alongside it. Log tailing + continues afterwards exactly as before. + """ + process = subprocess.Popen( + ["docker", "logs", "-f", container_id], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + served = not serve + try: + for line in process.stdout: + print(line, end="") + if not served and _WORKFLOW_UP_MARKER in line: + served = True + print("\nWorkflow is up -- starting the local dashboard (canyonos serve)...") + try: + run_serve() + except Exception as e: + print(f"Could not start the dashboard automatically: {e}") + print("Run `canyonos serve` manually to view it.") + except KeyboardInterrupt: + print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") + print("To resubscribe to log stream run `canyonos logs`.") + finally: + if process.poll() is None: + process.terminate() diff --git a/cli/canyonos/init.py b/cli/canyonos/init.py new file mode 100644 index 0000000..3d9cee9 --- /dev/null +++ b/cli/canyonos/init.py @@ -0,0 +1,129 @@ +""" +Logic for `canyonos init`, does the following: +1. Pull the Global Controller image +2. Start a container from it +3. Record where it's listening so cli knows where to send requests. +""" + +import json +import os +import subprocess +import urllib.error +import urllib.request + +# Formatting +from pyfiglet import figlet_format +from rich.console import Console + +from canyonos.theme import GRADIENT + + + +# Image Name, need to switch to CanyonCore Organization Namespace later +GC_IMAGE = "saakeths/canyonos:latest" +GC_CONTAINER_PORT = 8000 + +# Named docker volume mounted at /workspace inside the container. Unlike a bind +# mount, this lives in the container's docker volume (not the host filesystem): +# it persists across `canyonos quit` (docker rm leaves named volumes intact) and +# is unaffected by host-side changes. Files are copied in via `canyonos sync` +# (docker cp), not mounted live. +GC_WORKSPACE_VOLUME = "canyonos-workspace" +GC_WORKSPACE_PATH = "/workspace" + +STATE_DIR = os.path.expanduser("~/.canyonos") +STATE_PATH = os.path.join(STATE_DIR, "state.json") + + +def pull_image(image=GC_IMAGE): + # Capture output so the rich status spinner isn't clobbered by docker's own + # layer-progress printing. + subprocess.run(["docker", "pull", image], check=True, capture_output=True) + + +def _port_reachable(port, attempts=10, delay=0.5): + """ + A successful `docker run` only means Docker accepted the port binding -- + not that traffic actually flows. OrbStack's own port-forwarding proxy for + a given port can get stuck (heavy churn on the same port is enough to + trigger it), which looks fine at the Docker level but resets every real + connection. Confirm the container is actually reachable before trusting it. + """ + import time + + url = f"http://127.0.0.1:{port}/status" + for _ in range(attempts): + try: + urllib.request.urlopen(url, timeout=1) + return True + except (urllib.error.URLError, OSError): + time.sleep(delay) + return False + + +def run_container(image=GC_IMAGE, max_attempts=50): + port = GC_CONTAINER_PORT + for _ in range(max_attempts): + result = subprocess.run( + [ + "docker", + "run", + "-d", + "-p", + f"127.0.0.1:{port}:{GC_CONTAINER_PORT}", + # Docker-outside-of-Docker: GC shells out to `docker` to launch + # Redis/agent containers, so it needs the host's real daemon, + # not a nested one. + "-v", + "/var/run/docker.sock:/var/run/docker.sock", + "-v", + f"{GC_WORKSPACE_VOLUME}:{GC_WORKSPACE_PATH}", + "--add-host=host.docker.internal:host-gateway", + "-e", + "VENTIS_REDIS_HOST=host.docker.internal", + image, + ], + capture_output=True, + text=True, + ) + if result.returncode == 0: + container_id = result.stdout.strip() + if _port_reachable(port): + return container_id, port + # Port bound fine but never actually became reachable -- treat + # like a conflict, since that's effectively what it is. + subprocess.run(["docker", "rm", "-f", container_id], capture_output=True) + port += 1 + continue + if "port is already allocated" in result.stderr: + port += 1 + continue + raise RuntimeError(result.stderr) + raise RuntimeError(f"no free port found after {max_attempts} attempts starting at {GC_CONTAINER_PORT}") + + +def save_state(container_id, port): + os.makedirs(STATE_DIR, exist_ok=True) + with open(STATE_PATH, "w") as f: + json.dump({"container_id": container_id, "port": port}, f) + + +def load_state(): + with open(STATE_PATH) as f: + return json.load(f) + + +def run_init(): + console = Console() + banner = figlet_format("CANYON OS", font="ansi_shadow", width=200) + + for line, color in zip(banner.splitlines(), GRADIENT): + console.print(line, style=color) + + + with console.status("Pulling Global Controller image..."): + pull_image() + with console.status("Starting Global Controller container..."): + container_id, port = run_container() + save_state(container_id, port) + print(f"Global Controller running in container {container_id[:12]} on port {port}") diff --git a/cli/canyonos/integrate.py b/cli/canyonos/integrate.py new file mode 100644 index 0000000..23e61b1 --- /dev/null +++ b/cli/canyonos/integrate.py @@ -0,0 +1,82 @@ +""" +Logic for `canyonos integrate`: install the CanyonOS skill on a coding agent, +then launch that agent with a prompt to apply it to the current project. +""" + +import os +import shutil +import subprocess + +from rich.console import Console + +from utils.tui import select_menu + +# Points at the skill's folder, so SKILL.md and references/ both come along. +SKILL_SOURCE_URL = "https://github.com/CanyonCodeCoreAI/canyoncodecore/tree/nickhuo/porting-skill-car-layout/.claude/skills/porting-to-canyonos" + +# The porting skill emits no otel config; without this the dashboard stays empty. +OTEL_BLOCK = """otel: + destinations: + - name: local + protocol: http + endpoint: http://host.docker.internal:3000/v1/traces + headers: {}""" + +INTEGRATE_PROMPT = ( + "Use the CanyonOS porting-to-canyonos-core skill to convert the codebase in this directory to a canyonos-compatable format. No changes should be made to the current files, but all modifications should be put into a new .car folder." + "\n\nFinally, add the following block verbatim to the generated config/global_controller.yaml," + " at the top level as a sibling of `agents:`. Copy it exactly -- `protocol` must be http, and" + " the endpoint must keep the /v1/traces path:\n\n" + OTEL_BLOCK +) + +AGENTS = { + "claude": { + "label": "Claude Code", + "cli": "claude", + # Claude Code auto-loads project-local skills from here. + "skill_dir": ".claude/skills/porting-to-canyonos-core", + }, + "codex": { + "label": "Codex", + "cli": "codex", + # Codex only auto-loads skills from the user's home directory, not per-project. + "skill_dir": os.path.expanduser("~/.codex/skills/porting-to-canyonos-core"), + }, +} + + +def prompt_agent(): + options = [(key, spec["label"]) for key, spec in AGENTS.items()] + return select_menu(options, title="Which coding agent do you want to integrate with?") + + +def install_skill(agent): + spec = AGENTS[agent] + # -f overwrites an existing skill dir; without it gitpick exits 1 when the + # target already exists and is non-empty (e.g. re-running `integrate`). + subprocess.run( + ["npx", "-y", "gitpick", "-f", SKILL_SOURCE_URL, spec["skill_dir"]], + check=True, + ) + + +def launch_agent(agent, prompt): + spec = AGENTS[agent] + if not shutil.which(spec["cli"]): + print(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") + return + subprocess.run([spec["cli"], prompt], check=True) + + +def run_integrate(): + console = Console() + agent = prompt_agent() + if agent is None: + console.print("Cancelled.") + return + + console.print(f"Installing CanyonOS skill for {AGENTS[agent]['label']}...") + install_skill(agent) + + console.print(f"Launching {AGENTS[agent]['label']}...") + launch_agent(agent, INTEGRATE_PROMPT) diff --git a/cli/canyonos/logs.py b/cli/canyonos/logs.py new file mode 100644 index 0000000..9b2b1d8 --- /dev/null +++ b/cli/canyonos/logs.py @@ -0,0 +1,37 @@ +""" +Logic for `canyonos logs`: re-subscribe to the running deploy's log stream. +""" + +import json +import subprocess +import urllib.error +import urllib.request + +from canyonos.init import load_state + + +def run_logs(): + try: + state = load_state() + except FileNotFoundError: + print("No Global Controller container is running. Run `canyonos init` first.") + return + + url = f"http://127.0.0.1:{state['port']}/status" + req = urllib.request.Request(url, method="GET") + + try: + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + except urllib.error.URLError as e: + print(f"Could not reach Global Controller container: {e}") + return + + if not data.get("running"): + print("No deploy running, run `canyonos deploy` to deploy project.") + return + + try: + subprocess.run(["docker", "logs", "-f", state["container_id"]]) + except KeyboardInterrupt: + print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") diff --git a/cli/canyonos/new_app.py b/cli/canyonos/new_app.py new file mode 100644 index 0000000..30e93b0 --- /dev/null +++ b/cli/canyonos/new_app.py @@ -0,0 +1,21 @@ +""" +Logic for `canyonos new-app`: scaffold a new project in the current +directory. Runs locally, no container involved. +""" + +import os + + +def run_new_app(): + if os.listdir("."): + print("Directory is not empty. Run `canyonos new-app` in an empty directory.") + return + + for folder in ("agents", "config", "workflow"): + os.makedirs(folder) + open(".env", "w").close() + + for filename in ("global_controller.yaml", "policy.yaml"): + open(os.path.join("config", filename), "w").close() + + print("Created new CanyonOS project.") diff --git a/cli/canyonos/quit.py b/cli/canyonos/quit.py new file mode 100644 index 0000000..15aff2c --- /dev/null +++ b/cli/canyonos/quit.py @@ -0,0 +1,63 @@ +""" +Logic for `canyonos quit`: full teardown. Stops and removes the Global +Controller container AND deletes the /workspace named volume, so the project +files copied into it are discarded too. (Use `canyonos stop` to only halt a +running deploy while keeping the container and files around.) +""" + +import os +import subprocess + +from rich.console import Console + +from canyonos.init import GC_WORKSPACE_VOLUME, STATE_PATH, load_state +from canyonos.stop import _post_clean + + +def _container_exists(container_id): + result = subprocess.run( + ["docker", "inspect", container_id], capture_output=True + ) + return result.returncode == 0 + + +def run_quit(): + try: + state = load_state() + except FileNotFoundError: + print("No Global Controller container is running.") + return + + container_id = state["container_id"] + console = Console() + with console.status("Tearing down..."): + # Stop any running deploy first, so the local controller and Redis + # containers it spawned via docker-outside-of-docker get torn down + # too. Removing the GC container itself doesn't touch them -- they're + # sibling containers on the host, not nested inside it. + try: + _post_clean(state["port"]) + except OSError: + # Covers urllib.error.HTTPError/URLError (both subclass OSError) + # plus raw connection errors -- nothing was running, or the GC is + # already unreachable/gone. + pass + + # state.json can go stale (daemon restarted, container removed by + # hand, a previous `quit` died partway through) -- don't let a + # missing container turn `quit` into a crash instead of a cleanup. + already_gone = not _container_exists(container_id) + if not already_gone: + subprocess.run(["docker", "stop", container_id], check=False, capture_output=True) + subprocess.run(["docker", "rm", container_id], check=False, capture_output=True) + + # Remove the workspace volume only after the container is gone (docker + # refuses to remove a volume still in use). check=False so a missing + # volume doesn't turn teardown into an error. + subprocess.run(["docker", "volume", "rm", GC_WORKSPACE_VOLUME], check=False, capture_output=True) + os.remove(STATE_PATH) + + if already_gone: + print(f"Global Controller container {container_id[:12]} was already gone; cleaned up local state.") + else: + print(f"Global Controller container {container_id[:12]} torn down (volume removed)") diff --git a/cli/canyonos/serve.py b/cli/canyonos/serve.py new file mode 100644 index 0000000..ddfd7d6 --- /dev/null +++ b/cli/canyonos/serve.py @@ -0,0 +1,18 @@ +"""CLI output for the local dashboard stack.""" + +from .dashboard_stack import run_dashboard + + +def run_serve(config_path: str | None = None) -> int: + def report(phase: str, message: str) -> None: + print(f"[serve] {phase}: {message}") + + result = run_dashboard(config_path, report) + if result.ok: + print(f"Dashboard: {result.url}") + return 0 + + print(f"serve failed in {result.phase}: {result.message}") + if result.log_path: + print(f"log: {result.log_path}") + return 1 diff --git a/cli/canyonos/stop.py b/cli/canyonos/stop.py new file mode 100644 index 0000000..2f6ad40 --- /dev/null +++ b/cli/canyonos/stop.py @@ -0,0 +1,47 @@ +""" +Logic for `canyonos stop`: stop the running deploy inside the Global +Controller container (SIGTERM, same teardown as Ctrl+C would trigger). +""" + +import json +import urllib.error +import urllib.request + +from rich.console import Console + +from canyonos.init import load_state + + +def _post_clean(port): + """POST /clean to the Global Controller container. + + This is what actually tears down the local controller and Redis + containers a deploy spawned via docker-outside-of-docker: it sends + SIGTERM to the in-container `ventis deploy` process, whose handler calls + `GlobalController.stop()` and blocks until it returns. Shared with + `canyonos quit`, which needs the same teardown before removing the GC + container itself. + """ + url = f"http://127.0.0.1:{port}/clean" + req = urllib.request.Request(url, method="POST") + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + +def run_stop(): + try: + state = load_state() + except FileNotFoundError: + print("No Global Controller container is running. Run `canyonos init` first.") + return + + console = Console() + try: + with console.status("Stopping deploy..."): + _post_clean(state["port"]) + print("Deploy stopped.") + except urllib.error.HTTPError as e: + data = json.loads(e.read()) + print(f"Stop failed: {data.get('error')}") + except urllib.error.URLError as e: + print(f"Could not reach Global Controller container: {e}") diff --git a/cli/canyonos/sync.py b/cli/canyonos/sync.py new file mode 100644 index 0000000..f350a1c --- /dev/null +++ b/cli/canyonos/sync.py @@ -0,0 +1,40 @@ +""" +Logic for `canyonos sync`: copy the current project directory into the Global +Controller container's /workspace volume via `docker cp`. + +Files live inside the container's named volume (see `init.py`), not on a live +bind mount -- so they persist across `canyonos quit` and survive host-side +changes. `docker cp` is additive: it overwrites/adds files but never deletes, +so build outputs generated inside the container (stubs/, grpc_stubs/, +docker_container/) survive a re-sync of the host source. +""" + +import os +import subprocess + +from canyonos.init import GC_WORKSPACE_PATH, load_state + + +def run_sync(): + """Copy the current directory into the container. Returns True on success.""" + try: + state = load_state() + except FileNotFoundError: + print("No Global Controller container is running. Run `canyonos init` first.") + return False + + container_id = state["container_id"] + # Trailing "/." copies the *contents* of the current directory into + # /workspace, rather than nesting it under /workspace/. + src = os.path.join(os.getcwd(), ".") + print(f"Syncing {os.getcwd()} -> {container_id[:12]}:{GC_WORKSPACE_PATH} ...") + + result = subprocess.run( + ["docker", "cp", src, f"{container_id}:{GC_WORKSPACE_PATH}"] + ) + if result.returncode != 0: + print("Sync failed.") + return False + + print("Sync complete.") + return True diff --git a/cli/canyonos/theme.py b/cli/canyonos/theme.py new file mode 100644 index 0000000..e74a069 --- /dev/null +++ b/cli/canyonos/theme.py @@ -0,0 +1,20 @@ +""" +CanyonOS standard color palette. + +The green->white gradient introduced by the `canyonos init` banner, reused +across the CLI so everything shares one look. `GREEN` is the primary brand +color; `WHITE` the secondary; `GRADIENT` the full ramp for multi-line output. +""" + +GREEN = "#2BD17E" +WHITE = "#FFFFFF" + +# Primary -> secondary ramp (used for the init banner, top to bottom). +GRADIENT = [ + "#2BD17E", + "#55DA98", + "#80E3B2", + "#AAEDCB", + "#D5F6E5", + "#FFFFFF", +] diff --git a/cli/cli.py b/cli/cli.py new file mode 100644 index 0000000..4c78043 --- /dev/null +++ b/cli/cli.py @@ -0,0 +1,217 @@ +""" +Most of the commands will be executed by code in the canyonos container. +Anything executing in this CLI pertains to file/folder modification +""" + +import argparse +import sys + +from canyonos.clean import run_clean +from canyonos.constants import default_config_path +from canyonos.config import run_config +from canyonos.deploy import run_deploy +from canyonos.integrate import run_integrate +from canyonos.logs import run_logs +from canyonos.new_app import run_new_app +from canyonos.quit import run_quit +from canyonos.serve import run_serve +from canyonos.stop import run_stop +from canyonos.sync import run_sync + +try: + from rich.console import Console + from rich.panel import Panel + from rich.text import Text + from rich.table import Table + RICH_AVAILABLE = True +except ImportError: + RICH_AVAILABLE = False + +def cmd_connect(args): + pass + +def cmd_quit(args): + run_quit() + +def cmd_new_app(args): + run_new_app() + +# Executed in canyonos: syncs files, then builds + deploys +def cmd_deploy(args): + run_deploy(args.config, serve=args.serve) + +def cmd_clean(args): + run_clean() + +def cmd_stop(args): + run_stop() + +def cmd_logs(args): + run_logs() + +def cmd_sync(args): + run_sync() + +def cmd_config(args): + run_config() + +def cmd_integrate(args): + run_integrate() + +def cmd_doctor(args): + pass + +def cmd_serve(args): + sys.exit(run_serve(args.config)) + +# Executed in canyonos +def cmd_test(args): + pass + +# Executed in canyonos +def cmd_mega_build(args): + pass + + +def cmd_version(args): + pass + + +def print_custom_help(): + """Print a custom, visually appealing help screen.""" + if RICH_AVAILABLE: + console = Console() + + # Header + title = Text("CanyonOS CLI", style="bold cyan") + subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease\n", style="dim") + + console.print(Panel(title + subtitle, border_style="cyan", padding=(1, 2))) + + # Core commands + console.print("\n[bold yellow]Core Commands[/bold yellow]") + core_table = Table(show_header=False, border_style="dim", padding=(0, 2)) + core_table.add_column(style="cyan", width=20) + core_table.add_column(style="white") + core_table.add_row("integrate", "Sync source files to .car/app/") + core_table.add_row("deploy", "Build and deploy agents to configured hosts") + core_table.add_row("config", "Configure project settings") + console.print(core_table) + + # Utils commands + console.print("\n[bold yellow]Utils[/bold yellow]") + utils_table = Table(show_header=False, border_style="dim", padding=(0, 2)) + utils_table.add_column(style="cyan", width=20) + utils_table.add_column(style="white") + utils_table.add_row("new-app", "Create a new CanyonOS project") + utils_table.add_row("serve", "Start local CanyonOS dashboard") + utils_table.add_row("sync", "Sync files with container") + utils_table.add_row("stop", "Stop running containers") + utils_table.add_row("clean", "Remove generated files") + utils_table.add_row("logs", "View container logs") + utils_table.add_row("doctor", "Check system health") + utils_table.add_row("connect", "Connect to remote host") + utils_table.add_row("quit", "Shut down CanyonOS services") + console.print(utils_table) + + # Quick start + console.print("\n[bold green]Quick Start:[/bold green]") + console.print(" [dim]1.[/dim] canyonos new-app [cyan]my-app[/cyan]") + console.print(" [dim]2.[/dim] cd [cyan]my-app[/cyan]") + console.print(" [dim]3.[/dim] canyonos integrate") + console.print(" [dim]4.[/dim] canyonos deploy") + console.print(" [dim]5.[/dim] canyonos serve\n") + + console.print("[dim]For command-specific help: [cyan]canyonos --help[/cyan][/dim]\n") + else: + # Fallback to simple text if rich is not available + print("\n" + "="*60) + print(" " * 20 + "CanyonOS CLI") + print(" " * 10 + "Build, deploy, and manage agentic workflows") + print("="*60 + "\n") + + print("CORE COMMANDS:") + print(" integrate Sync source files to .car/app/") + print(" deploy Build and deploy agents to configured hosts") + print(" config Configure project settings\n") + + print("UTILS:") + print(" new-app Create a new CanyonOS project") + print(" serve Start local CanyonOS dashboard") + print(" sync Sync files with container") + print(" stop Stop running containers") + print(" clean Remove generated files") + print(" logs View container logs") + print(" doctor Check system health") + print(" connect Connect to remote host") + print(" quit Shut down CanyonOS services\n") + + print("QUICK START:") + print(" 1. canyonos new-app my-app") + print(" 2. cd my-app") + print(" 3. canyonos integrate") + print(" 4. canyonos deploy") + print(" 5. canyonos serve\n") + + print("For command-specific help: canyonos --help\n") + + +def _parse_bool(value): + if value.lower() in ("true", "1", "yes"): + return True + if value.lower() in ("false", "0", "no"): + return False + raise argparse.ArgumentTypeError(f"expected true/false, got: {value!r}") + + +def main(): + parser = argparse.ArgumentParser(prog="canyonos") + subparsers = parser.add_subparsers(dest="command") + config_default = default_config_path() + + subparsers.add_parser("new-app").set_defaults(func=cmd_new_app) + deploy = subparsers.add_parser("deploy") + deploy.add_argument( + "-c", + "--config", + default=config_default, + help=f"Path to global controller config (default: {config_default})", + ) + deploy.add_argument( + "--serve", + type=_parse_bool, + default=True, + metavar="true|false", + help="Automatically launch the local dashboard (canyonos serve) once the workflow is up (default: true)", + ) + deploy.set_defaults(func=cmd_deploy) + subparsers.add_parser("clean").set_defaults(func=cmd_clean) + subparsers.add_parser("stop").set_defaults(func=cmd_stop) + subparsers.add_parser("logs").set_defaults(func=cmd_logs) + subparsers.add_parser("quit").set_defaults(func=cmd_quit) + subparsers.add_parser("connect").set_defaults(func=cmd_connect) + subparsers.add_parser("sync").set_defaults(func=cmd_sync) + subparsers.add_parser("config").set_defaults(func=cmd_config) + subparsers.add_parser("integrate").set_defaults(func=cmd_integrate) + subparsers.add_parser("doctor").set_defaults(func=cmd_doctor) + serve = subparsers.add_parser("serve") + serve.add_argument( + "-c", + "--config", + default=config_default, + help=f"Path to global controller config (default: {config_default})", + ) + serve.set_defaults(func=cmd_serve) + subparsers.add_parser("test").set_defaults(func=cmd_test) + subparsers.add_parser("mega-build").set_defaults(func=cmd_mega_build) + + args = parser.parse_args() + if not getattr(args, "command", None): + print_custom_help() + return + + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/cli/pyproject.toml b/cli/pyproject.toml new file mode 100644 index 0000000..e85c825 --- /dev/null +++ b/cli/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "canyonos" +version = "0.1.4" +description = "CanyonOS CLI" +requires-python = ">=3.10" +dependencies = [ + "pyfiglet", + "pyyaml", + "rich", + "ruamel.yaml", +] + +[project.scripts] +canyonos = "cli:main" + +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["canyonos*", "utils*"] + +[tool.setuptools] +py-modules = ["cli"] + +[tool.setuptools.package-data] +canyonos = ["dashboard.compose.yml"] diff --git a/cli/tests/test_dashboard_stack.py b/cli/tests/test_dashboard_stack.py new file mode 100644 index 0000000..2e62480 --- /dev/null +++ b/cli/tests/test_dashboard_stack.py @@ -0,0 +1,432 @@ +import json +import subprocess +from pathlib import Path + +import pytest + +from canyonos import dashboard_stack + + +def completed(argv, returncode=0, stdout="", stderr=""): + return subprocess.CompletedProcess(argv, returncode, stdout, stderr) + + +@pytest.fixture +def project(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + for key in ( + "DATABASE_URL", + "JWT_SECRET", + "CANYONOS_DATABASE_URL", + "CANYONOS_JWT_SECRET", + "CANYONOS_REDIS_HOST", + "CANYONOS_REDIS_PORT", + "CANYONOS_API_IMAGE", + "CANYONOS_WEB_IMAGE", + ): + monkeypatch.delenv(key, raising=False) + config_dir = tmp_path / "config" + config_dir.mkdir() + config = config_dir / "global_controller.yaml" + config.write_text("database:\n url: postgres://user:password@db.example/canyonos\n") + monkeypatch.setattr(dashboard_stack.shutil, "which", lambda _: "/usr/bin/docker") + monkeypatch.setattr(dashboard_stack, "_port_is_free", lambda _port: True) + return config + + +def install_docker(monkeypatch, calls, responses=None): + responses = responses or {} + + def fake_run(argv, **_): + calls.append(argv) + if callable(responses): + return responses(argv) + for marker, result in responses.items(): + if argv[-len(marker) :] == list(marker): + return result(argv) + if argv[:3] == ["docker", "container", "inspect"]: + return completed(argv, returncode=1) + return completed(argv) + + monkeypatch.setattr(dashboard_stack.subprocess, "run", fake_run) + + +@pytest.mark.parametrize( + ("prepare", "message"), + [ + (lambda monkeypatch, _: monkeypatch.setattr(dashboard_stack.shutil, "which", lambda _: None), "docker is not on PATH"), + ( + lambda _, responses: responses.update( + {("info",): lambda argv: completed(argv, returncode=1)} + ), + "docker daemon or socket is unavailable", + ), + ( + lambda _, responses: responses.update( + {("compose", "version"): lambda argv: completed(argv, returncode=1)} + ), + "docker compose is unavailable", + ), + ], +) +def test_docker_validation_failures_do_not_pull(monkeypatch, project, prepare, message): + calls = [] + responses = {} + prepare(monkeypatch, responses) + install_docker(monkeypatch, calls, responses) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result == dashboard_stack.ServeResult(False, "validate", message) + assert all(command[-1] != "pull" for command in calls) + + +def test_empty_database_url_fails_validation_but_absent_one_does_not(monkeypatch, project): + project.write_text("database:\n url: ''\n") + calls = [] + install_docker(monkeypatch, calls) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result == dashboard_stack.ServeResult( + False, "validate", "database.url must be a non-empty string" + ) + assert all(command[-1] != "pull" for command in calls) + + +def test_missing_database_section_is_not_a_validation_failure(monkeypatch, project): + project.write_text("") + calls = [] + install_docker(monkeypatch, calls) + + stack = dashboard_stack.validate(str(project)) + + assert stack.database_url is None + + +def test_prepare_omits_database_env_when_not_configured(project): + project.write_text("") + stack = dashboard_stack.DashboardStack( + None, dashboard_stack._state_dir(), Path.cwd() + ) + + managed_env, message = dashboard_stack.prepare(stack) + + assert message == "dashboard state prepared" + assert "CANYONOS_DATABASE_URL" not in managed_env + assert "CANYONOS_DATABASE_URL" not in stack.env_path.read_text() + + +def test_config_substitutes_quoted_dotenv_value_without_replacing_source(monkeypatch, project): + source_line = 'DATABASE_URL="postgres://user:password@db.example/canyonos"\n' + Path.cwd().joinpath(".env").write_text(source_line) + project.write_text("database:\n url: ${DATABASE_URL}\n") + calls = [] + install_docker(monkeypatch, calls) + + stack = dashboard_stack.validate(str(project)) + managed_env, _ = dashboard_stack.prepare(stack) + + assert stack.database_url == "postgres://user:password@db.example/canyonos" + assert managed_env["CANYONOS_DATABASE_URL"] == stack.database_url + assert stack.env_path.read_text().startswith(source_line) + + +def test_missing_database_url_variable_fails_without_pulling(monkeypatch, project): + project.write_text("database:\n url: ${DATABASE_URL}\n") + calls = [] + install_docker(monkeypatch, calls) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result == dashboard_stack.ServeResult( + False, + "validate", + "database.url needs ${DATABASE_URL}, which is not set in the project .env", + ) + assert all(command[-1] != "pull" for command in calls) + + +def test_user_jwt_secret_is_untouched_while_canyonos_secret_is_stable(project): + source_line = "JWT_SECRET=user-value\n" + Path.cwd().joinpath(".env").write_text(source_line) + stack = dashboard_stack.DashboardStack( + "postgres://user:password@db.example/canyonos", + dashboard_stack._state_dir(), + Path.cwd(), + ) + + first_env, _ = dashboard_stack.prepare(stack) + second_env, _ = dashboard_stack.prepare(stack) + + env_contents = stack.env_path.read_text() + assert env_contents.startswith(source_line) + assert first_env["CANYONOS_JWT_SECRET"] != "user-value" + assert first_env["CANYONOS_JWT_SECRET"] == second_env["CANYONOS_JWT_SECRET"] + + +def test_process_environment_database_url_wins_over_project_dotenv(monkeypatch, project): + Path.cwd().joinpath(".env").write_text("DATABASE_URL=postgres://from-file/canyonos\n") + monkeypatch.setenv("DATABASE_URL", "postgres://from-process/canyonos") + project.write_text("database:\n url: ${DATABASE_URL}\n") + calls = [] + install_docker(monkeypatch, calls) + + stack = dashboard_stack.validate(str(project)) + + assert stack.database_url == "postgres://from-process/canyonos" + + +def test_unreadable_config_does_not_pull(monkeypatch, project): + calls = [] + install_docker(monkeypatch, calls) + + result = dashboard_stack.run_dashboard(str(project.with_name("missing.yaml"))) + + assert result.message == f"config file is not readable: {project.with_name('missing.yaml')}" + assert all(command[-1] != "pull" for command in calls) + + +def test_state_directory_and_port_validation_failures_do_not_pull(monkeypatch, project, tmp_path): + calls = [] + install_docker(monkeypatch, calls) + blocked_state_dir = tmp_path / "blocked" + blocked_state_dir.write_text("not a directory") + monkeypatch.setattr(dashboard_stack, "_state_dir", lambda: blocked_state_dir) + + state_result = dashboard_stack.run_dashboard(str(project)) + + assert state_result.message == "dashboard state directory is not writable" + assert all(command[-1] != "pull" for command in calls) + + calls.clear() + monkeypatch.setattr(dashboard_stack, "_state_dir", lambda: tmp_path / "state") + monkeypatch.setattr(dashboard_stack, "_find_web_port", lambda start=8080, max_attempts=50: (_ for _ in ()).throw( + dashboard_stack.PhaseFailure("validate", "no free port found for the dashboard after 50 attempts starting at 8080") + )) + port_result = dashboard_stack.run_dashboard(str(project)) + + assert port_result.message == "no free port found for the dashboard after 50 attempts starting at 8080" + assert all(command[-1] != "pull" for command in calls) + + +def test_prepare_preserves_unrelated_env_lines_and_mode(project): + Path.cwd().joinpath(".env").write_text( + "OTHER=one\n# preserved\nJWT_SECRET=kept-secret\nLAST=two\n" + ) + stack = dashboard_stack.DashboardStack( + "postgres://user:password@localhost:5432/canyonos", + dashboard_stack._state_dir(), + Path.cwd(), + ) + + managed_env, message = dashboard_stack.prepare(stack) + + assert message == "database host localhost is reachable from the stack as host.docker.internal" + env_lines = stack.env_path.read_text().splitlines() + assert env_lines[:2] == ["OTHER=one", "# preserved"] + assert env_lines[2] == "JWT_SECRET=kept-secret" + assert env_lines[3] == "LAST=two" + assert {line.split("=", 1)[0] for line in env_lines if "=" in line} == { + "OTHER", + "JWT_SECRET", + "LAST", + "CANYONOS_DATABASE_URL", + "CANYONOS_JWT_SECRET", + "CANYONOS_REDIS_HOST", + "CANYONOS_REDIS_PORT", + "CANYONOS_API_IMAGE", + "CANYONOS_WEB_IMAGE", + "CANYONOS_WEB_PORT", + } + assert "CANYONOS_DATABASE_URL=postgres://user:password@host.docker.internal:5432/canyonos" in env_lines + assert stack.env_path.stat().st_mode & 0o777 == 0o600 + assert stack.state_dir.stat().st_mode & 0o777 == 0o700 + assert sorted(path.name for path in stack.state_dir.iterdir()) == ["stack.json"] + + +def test_prepare_reuses_secret_and_rewrites_only_local_hosts(project): + stack = dashboard_stack.DashboardStack( + "postgres://user:password@localhost/canyonos", + dashboard_stack._state_dir(), + Path.cwd(), + ) + first_env, first_message = dashboard_stack.prepare(stack) + second_env, second_message = dashboard_stack.prepare(stack) + + assert first_message.startswith("database host localhost") + assert second_message.startswith("database host localhost") + assert first_env["CANYONOS_JWT_SECRET"] == second_env["CANYONOS_JWT_SECRET"] + assert ( + first_env["CANYONOS_DATABASE_URL"] + == "postgres://user:password@host.docker.internal/canyonos" + ) + + remote_stack = dashboard_stack.DashboardStack( + "postgres://db.example/canyonos", dashboard_stack._state_dir(), Path.cwd() + ) + remote_env, _ = dashboard_stack.prepare(remote_stack) + assert remote_env["CANYONOS_DATABASE_URL"] == "postgres://db.example/canyonos" + + +def test_redaction_removes_urls_secrets_and_credentials(): + database_url = "postgres://user:password@db.example/canyonos" + secret = "secret-value" + logs = f"{database_url}\n{secret}\nredis://other:credential@cache:6379/0" + + redacted = dashboard_stack.redact_logs(logs, secret) + + assert database_url not in redacted # credentials portion is stripped by the generic regex + assert secret not in redacted + assert "user:password@" not in redacted + assert "other:credential@" not in redacted + + +@pytest.mark.parametrize("had_containers", [False, True]) +def test_start_failure_saves_log_and_cleans_up_only_new_stack(monkeypatch, project, had_containers): + calls = [] + + def response(argv): + if argv[-2:] == ["ps", "-q"]: + return completed(argv, stdout="existing\n" if had_containers else "") + if argv[-5:] == ["up", "-d", "--wait", "--wait-timeout", "180"]: + return completed(argv, returncode=1) + if argv[-4:] == ["logs", "--no-color", "--tail", "200"]: + return completed(argv, stdout="postgres://user:password@db.example/canyonos") + return completed(argv) + + install_docker(monkeypatch, calls, response) + result = dashboard_stack.run_dashboard(str(project)) + + assert result.ok is False + assert result.phase == "start" + assert result.log_path is not None + assert Path(result.log_path).stat().st_mode & 0o777 == 0o600 + assert "postgres://user:password@db.example/canyonos" not in Path(result.log_path).read_text() + assert any("logs" in command for command in calls) + assert any(command[-1] == "down" for command in calls) is (not had_containers) + + +def test_pull_failure_includes_redacted_stderr(monkeypatch, project): + calls = [] + secret = None + + def response(argv): + nonlocal secret + if argv[-1] == "pull": + secret = next( + line.split("=", 1)[1] + for line in Path.cwd().joinpath(".env").read_text().splitlines() + if line.startswith("CANYONOS_JWT_SECRET=") + ) + return completed( + argv, + returncode=1, + stderr=f"first line\npull unauthorized postgres://user:password@db.example/canyonos {secret}\n", + ) + return completed(argv) + + install_docker(monkeypatch, calls, response) + result = dashboard_stack.run_dashboard(str(project)) + + assert result.phase == "pull" + assert "pull unauthorized" in result.message + assert "postgres://user:password@db.example/canyonos" not in result.message + assert "user:password@" not in result.message + assert secret not in result.message + + +def test_verify_failure_saves_a_log(monkeypatch, project): + calls = [] + install_docker(monkeypatch, calls) + probes = [] + + def urlopen(*_args, **_kwargs): + probes.append(True) + raise dashboard_stack.urllib.error.URLError("down") + + monkeypatch.setattr( + dashboard_stack.urllib.request, + "urlopen", + urlopen, + ) + clock = iter([0, 0, 0, 31]) + monkeypatch.setattr(dashboard_stack.time, "monotonic", lambda: next(clock)) + monkeypatch.setattr(dashboard_stack.time, "sleep", lambda _: None) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result.ok is False + assert result.phase == "verify" + assert result.log_path is not None + assert Path(result.log_path).is_file() + assert probes == [True] + assert any("logs" in command for command in calls) + assert any(command[-1] == "down" for command in calls) + + +def test_success_pulls_starts_and_verifies(monkeypatch, project): + calls = [] + install_docker(monkeypatch, calls) + + class Response: + status = 200 + + def close(self): + pass + + endpoints = [] + + def urlopen(endpoint, timeout): + endpoints.append((endpoint, timeout)) + return Response() + + monkeypatch.setattr(dashboard_stack.urllib.request, "urlopen", urlopen) + result = dashboard_stack.run_dashboard(str(project)) + + assert result == dashboard_stack.ServeResult( + True, "verify", "dashboard health checks passed", "http://127.0.0.1:8080" + ) + pull_index = next(index for index, command in enumerate(calls) if command[-1] == "pull") + up_index = next(index for index, command in enumerate(calls) if "up" in command) + assert pull_index < up_index + assert calls[pull_index][4:6] == ["--env-file", str(project.parent.parent / ".env")] + assert calls[up_index][-5:] == ["up", "-d", "--wait", "--wait-timeout", "180"] + assert endpoints == [ + ("http://127.0.0.1:8080/healthz", 5), + ("http://127.0.0.1:8080/api/healthz", 5), + ] + + +def test_existing_dashboard_container_skips_port_check(monkeypatch, project): + calls = [] + + def response(argv): + if argv[:3] == ["docker", "container", "inspect"]: + return completed( + argv, + stdout=json.dumps( + [{"NetworkSettings": {"Ports": {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": "8080"}]}}}] + ), + ) + if argv[-2:] == ["ps", "-q"]: + return completed(argv, stdout="existing\n") + return completed(argv) + + install_docker(monkeypatch, calls, response) + monkeypatch.setattr( + dashboard_stack, + "_find_web_port", + lambda *a, **k: pytest.fail("the existing dashboard owns port 8080, should not search for a new one"), + ) + monkeypatch.setattr( + dashboard_stack.urllib.request, + "urlopen", + lambda *_args, **_kwargs: type("Response", (), {"status": 200, "close": lambda self: None})(), + ) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result.ok + assert result.url == "http://127.0.0.1:8080" diff --git a/cli/utils/__init__.py b/cli/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/utils/tui.py b/cli/utils/tui.py new file mode 100644 index 0000000..3fcdad5 --- /dev/null +++ b/cli/utils/tui.py @@ -0,0 +1,113 @@ +""" +Minimal arrow-key select menu, no dependency beyond the standard library. +""" + +import os +import select as select_syscall +import sys +import termios +import tty + +UP_KEYS = ("\x1b[A", "\x1bOA", "k") +DOWN_KEYS = ("\x1b[B", "\x1bOB", "j") +CANCEL_KEYS = ("\x03", "\x1b") +DELETE_KEYS = ("d", "D") +QUIT_KEYS = ("q", "Q") + +# Sentinel returned (paired with the hovered value) when the delete key is +# pressed and `deletable=True`. Callers check `result[0] is DELETE_ACTION`. +DELETE_ACTION = object() + +# Sentinel returned when the quit key is pressed and `quittable=True`. Distinct +# from None (which callers use for a single-level cancel/back) so a caller can +# unwind an entire nested session. Callers check `result is QUIT_ACTION`. +QUIT_ACTION = object() + + +def _read_key(fd): + # Reads straight off the fd (not sys.stdin) so this stays in sync with + # the select() call below -- stdin's own buffering can silently swallow + # an arrow key's trailing bytes before select() ever sees them queued. + ch = os.read(fd, 1).decode() + if ch == "\x1b": + # An arrow key arrives as a multi-byte escape sequence; a bare Esc + # press has nothing queued right behind it. + if select_syscall.select([fd], [], [], 0.01)[0]: + ch += os.read(fd, 1).decode() + if ch[-1] in ("[", "O"): + ch += os.read(fd, 1).decode() + return ch + + +def select_menu(options, title, deletable=False, quittable=False): + """Arrow-key single-select over `options` (a list of (value, label) pairs). + + Returns the chosen value, or None if there's nothing to choose from or + the user cancelled (Esc/Ctrl-C). + + If `deletable` is True, pressing the delete key ('d') over an item returns + the tuple `(DELETE_ACTION, hovered_value)` so the caller can act on the + currently-hovered item instead of selecting it. + + If `quittable` is True, pressing the quit key ('q') returns the sentinel + `QUIT_ACTION` -- distinct from None -- so the caller can unwind an entire + nested session rather than just this one menu. + """ + if len(options) == 1: + return options[0][0] + if not options or not sys.stdin.isatty(): + return None + + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + out = sys.stderr + idx = 0 + n = len(options) + + def frame(): + lines = [f"\x1b[1m{title}\x1b[0m", ""] + for i, (_, label) in enumerate(options): + lines.append(f"\x1b[36m❯ {label}\x1b[0m" if i == idx else f" {label}") + hint = "↑/↓ move · 1-9 jump · enter select" + if deletable: + hint += " · d delete" + if quittable: + hint += " · q quit" + hint += " · esc cancel" + lines.append(f"\x1b[2m{hint}\x1b[0m") + return "\r\n".join(lines) + + prev_frame = None + try: + tty.setraw(fd) + out.write("\x1b[?25l") + while True: + text = frame() + if prev_frame is not None: + # How far back up to move is read off the frame we actually + # wrote last time, not recomputed separately -- it can't drift + # out of sync with what's really on screen. + out.write(f"\r\x1b[{prev_frame.count(chr(10))}A\x1b[J") + out.write(text) + out.flush() + prev_frame = text + + key = _read_key(fd) + if key in ("\r", "\n"): + return options[idx][0] + if key in CANCEL_KEYS: + return None + if key in UP_KEYS: + idx = (idx - 1) % n + elif key in DOWN_KEYS: + idx = (idx + 1) % n + elif deletable and key in DELETE_KEYS: + return (DELETE_ACTION, options[idx][0]) + elif quittable and key in QUIT_KEYS: + return QUIT_ACTION + elif key.isdigit() and key != "0" and int(key) <= n: + return options[int(key) - 1][0] + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + out.write("\x1b[?25h\r\n") + out.flush() diff --git a/examples/epigenomics/README.md b/examples/epigenomics/README.md new file mode 100644 index 0000000..19fb748 --- /dev/null +++ b/examples/epigenomics/README.md @@ -0,0 +1,70 @@ +# Epigenomics Example + +A synthetic, LLM-free workflow modeled on the +[WfCommons Epigenomics recipe](https://docs.wfcommons.org/en/latest/generating_workflows.html): +a split → fan-out (filter → align → sort) → fan-in (dedup) → index pipeline. +Every stage does deterministic SHA-256 work sized off chunk byte counts, so +results are reproducible and the fan-out width scales with `num_chunks` -- +useful for exercising scheduling/replica behavior locally without any real +model calls. + +## Pipeline + +``` +SplitAgent --> FilterAgent --> MapAgent --> SortAgent --\ + (1 call) (fan-out) (fan-out) (fan-out) --> DedupAgent --> IndexAgent + (fan-in barrier) (1 call) +``` + +- **SplitAgent** — splits `input_size` bytes into `num_chunks` equal chunks. +- **FilterAgent** — per-chunk contaminant filter (light cost). +- **MapAgent** — per-chunk alignment, the heaviest stage. +- **SortAgent** — per-chunk sort (moderate cost). +- **DedupAgent** — merges every sorted chunk into one digest (the barrier). +- **IndexAgent** — builds the final index from the merged digest. + +## Quick Start + +```bash +# Build stubs and Docker images +ventis build + +# Launch all agents +ventis deploy + +# Test with curl +curl -X POST http://:8080/main \ + -H 'Content-Type: application/json' \ + -d '{"input_size": 65536, "num_chunks": 4}' + +# Check result +curl http://:8080/status/ +``` + +## Project Structure + +``` +├── agents/ # Agent implementations and YAML definitions +│ ├── split_agent.py/.yaml +│ ├── filter_agent.py/.yaml +│ ├── map_agent.py/.yaml +│ ├── sort_agent.py/.yaml +│ ├── dedup_agent.py/.yaml +│ └── index_agent.py/.yaml +├── workflow/ # Workflow script (deployed as a REST API) +│ └── epigenomics_workflow.py +└── config/ + ├── global_controller.yaml # Deployment configuration (provider: local) + └── policy.yaml # Access control rules +``` + +## Policy Rules + +Edit `config/policy.yaml` to control which callers can access which agents. +Pass `_context` in your curl request to set the caller identity: + +```bash +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' \ + -d '{"input_size": 65536, "num_chunks": 4, "_context": {"origin": "admin"}}' +``` diff --git a/examples/epigenomics/agents/dedup_agent.py b/examples/epigenomics/agents/dedup_agent.py new file mode 100644 index 0000000..d883854 --- /dev/null +++ b/examples/epigenomics/agents/dedup_agent.py @@ -0,0 +1,44 @@ +# Dedup Agent +# +# Fan-in barrier (mirrors Epigenomics' mark-duplicates/merge stage): needs +# every sorted chunk before it can run. Combines all chunk digests into one +# merged digest, with cost scaling off the total merged data volume. +# +# Resource profile: moderate CPU, single call per request (the barrier). + +import hashlib + + +class DedupAgent(object): + def __init__(self): + self.tools = [self.merge_dedup] + + def merge_dedup(self, chunks: list) -> dict: + """Merge and deduplicate every sorted chunk into one combined digest.""" + total_size = sum(c["size"] for c in chunks) + seed = "".join(c["digest"] for c in sorted(chunks, key=lambda c: c["chunk_id"])) + merged_digest = self._cpu_work(seed, total_size) + return { + "merged_digest": merged_digest, + "total_size": total_size, + "n_chunks": len(chunks), + } + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = DedupAgent() + print( + agent.merge_dedup( + [ + {"chunk_id": "chunk-0", "size": 16384, "digest": "aa"}, + {"chunk_id": "chunk-1", "size": 16384, "digest": "bb"}, + ] + ) + ) diff --git a/examples/epigenomics/agents/dedup_agent.yaml b/examples/epigenomics/agents/dedup_agent.yaml new file mode 100644 index 0000000..1c577cd --- /dev/null +++ b/examples/epigenomics/agents/dedup_agent.yaml @@ -0,0 +1,10 @@ +agent: + name: DedupAgent + functions: + - name: merge_dedup + description: Merge and deduplicate every sorted chunk into one combined digest. + arguments: + - name: chunks + type: list + returns: + type: dict diff --git a/examples/epigenomics/agents/filter_agent.py b/examples/epigenomics/agents/filter_agent.py new file mode 100644 index 0000000..24dfc48 --- /dev/null +++ b/examples/epigenomics/agents/filter_agent.py @@ -0,0 +1,32 @@ +# Filter Agent +# +# First per-chunk stage in the fan-out (mirrors Epigenomics' filter_contams +# stage): scrubs one chunk and hands back a content digest the later stages +# build on. The "work" is a deterministic SHA-256 chain sized off the +# chunk's declared byte size, standing in for the real stage's per-byte cost. +# +# Resource profile: light CPU, high fan-out (one call per chunk). + +import hashlib + + +class FilterAgent(object): + def __init__(self): + self.tools = [self.filter_contams] + + def filter_contams(self, chunk_id: str, size: int) -> dict: + """Filter contaminants out of one chunk, returning its content digest.""" + digest = self._cpu_work(chunk_id, size) + return {"chunk_id": chunk_id, "size": size, "digest": digest} + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = FilterAgent() + print(agent.filter_contams("chunk-0", 16384)) diff --git a/examples/epigenomics/agents/filter_agent.yaml b/examples/epigenomics/agents/filter_agent.yaml new file mode 100644 index 0000000..9f10d29 --- /dev/null +++ b/examples/epigenomics/agents/filter_agent.yaml @@ -0,0 +1,12 @@ +agent: + name: FilterAgent + functions: + - name: filter_contams + description: Filter contaminants out of one chunk, returning its content digest. + arguments: + - name: chunk_id + type: str + - name: size + type: int + returns: + type: dict diff --git a/examples/epigenomics/agents/index_agent.py b/examples/epigenomics/agents/index_agent.py new file mode 100644 index 0000000..6faa984 --- /dev/null +++ b/examples/epigenomics/agents/index_agent.py @@ -0,0 +1,30 @@ +# Index Agent +# +# Final stage (mirrors Epigenomics' index-build stage): produces the +# workflow's terminal artifact from the merged, deduplicated digest. +# +# Resource profile: light CPU, single call per request. + +import hashlib + + +class IndexAgent(object): + def __init__(self): + self.tools = [self.build_index] + + def build_index(self, merged_digest: str, total_size: int) -> dict: + """Build the final index from the merged digest.""" + index_digest = self._cpu_work(merged_digest, max(1, total_size // 4)) + return {"index_digest": index_digest, "total_size": total_size} + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = IndexAgent() + print(agent.build_index("deadbeef", 65536)) diff --git a/examples/epigenomics/agents/index_agent.yaml b/examples/epigenomics/agents/index_agent.yaml new file mode 100644 index 0000000..3424c44 --- /dev/null +++ b/examples/epigenomics/agents/index_agent.yaml @@ -0,0 +1,12 @@ +agent: + name: IndexAgent + functions: + - name: build_index + description: Build the final index from the merged digest. + arguments: + - name: merged_digest + type: str + - name: total_size + type: int + returns: + type: dict diff --git a/examples/epigenomics/agents/map_agent.py b/examples/epigenomics/agents/map_agent.py new file mode 100644 index 0000000..b9db3ef --- /dev/null +++ b/examples/epigenomics/agents/map_agent.py @@ -0,0 +1,33 @@ +# Map Agent +# +# Sequence-alignment stand-in (Epigenomics' map stage) -- by far the most +# CPU-expensive stage in the real workflow, so its per-byte cost multiplier +# here is set well above the other stages to match that shape. +# +# Resource profile: heavy CPU, high fan-out (one call per chunk). + +import hashlib + +COST_MULTIPLIER = 8 + + +class MapAgent(object): + def __init__(self): + self.tools = [self.align] + + def align(self, chunk_id: str, size: int, digest: str) -> dict: + """Align one filtered chunk, returning its post-alignment digest.""" + aligned = self._cpu_work(digest, size * COST_MULTIPLIER) + return {"chunk_id": chunk_id, "size": size, "digest": aligned} + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = MapAgent() + print(agent.align("chunk-0", 16384, "deadbeef")) diff --git a/examples/epigenomics/agents/map_agent.yaml b/examples/epigenomics/agents/map_agent.yaml new file mode 100644 index 0000000..66c1210 --- /dev/null +++ b/examples/epigenomics/agents/map_agent.yaml @@ -0,0 +1,14 @@ +agent: + name: MapAgent + functions: + - name: align + description: Align one filtered chunk, returning its post-alignment digest. + arguments: + - name: chunk_id + type: str + - name: size + type: int + - name: digest + type: str + returns: + type: dict diff --git a/examples/epigenomics/agents/sort_agent.py b/examples/epigenomics/agents/sort_agent.py new file mode 100644 index 0000000..2aa2df7 --- /dev/null +++ b/examples/epigenomics/agents/sort_agent.py @@ -0,0 +1,32 @@ +# Sort Agent +# +# Third per-chunk stage in the fan-out (mirrors Epigenomics' sort stage): +# orders one aligned chunk, returning an updated digest for the fan-in below. +# +# Resource profile: moderate CPU, high fan-out (one call per chunk). + +import hashlib + +COST_MULTIPLIER = 2 + + +class SortAgent(object): + def __init__(self): + self.tools = [self.sort] + + def sort(self, chunk_id: str, size: int, digest: str) -> dict: + """Sort one aligned chunk, returning its post-sort digest.""" + sorted_digest = self._cpu_work(digest, size * COST_MULTIPLIER) + return {"chunk_id": chunk_id, "size": size, "digest": sorted_digest} + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = SortAgent() + print(agent.sort("chunk-0", 16384, "deadbeef")) diff --git a/examples/epigenomics/agents/sort_agent.yaml b/examples/epigenomics/agents/sort_agent.yaml new file mode 100644 index 0000000..464dc85 --- /dev/null +++ b/examples/epigenomics/agents/sort_agent.yaml @@ -0,0 +1,14 @@ +agent: + name: SortAgent + functions: + - name: sort + description: Sort one aligned chunk, returning its post-sort digest. + arguments: + - name: chunk_id + type: str + - name: size + type: int + - name: digest + type: str + returns: + type: dict diff --git a/examples/epigenomics/agents/split_agent.py b/examples/epigenomics/agents/split_agent.py new file mode 100644 index 0000000..1b1a6be --- /dev/null +++ b/examples/epigenomics/agents/split_agent.py @@ -0,0 +1,27 @@ +# Split Agent +# +# Entry stage of the pipeline (mirrors WfCommons' Epigenomics fastq-split +# stage): splits one logical input into num_chunks equal-sized chunks for the +# downstream fan-out. There's no real sequence file here -- each chunk's +# "size" just stands in for its data volume, which is what every downstream +# stage prices its synthetic CPU work off of. +# +# Resource profile: cheap CPU, single call per request. + + +class SplitAgent(object): + def __init__(self): + self.tools = [self.split] + + def split(self, input_size: int, num_chunks: int) -> dict: + """Split input_size bytes of data into num_chunks equal chunks.""" + chunk_size = max(1, input_size // num_chunks) + chunks = [ + {"chunk_id": f"chunk-{i}", "size": chunk_size} for i in range(num_chunks) + ] + return {"chunks": chunks} + + +if __name__ == "__main__": + agent = SplitAgent() + print(agent.split(65536, 4)) diff --git a/examples/epigenomics/agents/split_agent.yaml b/examples/epigenomics/agents/split_agent.yaml new file mode 100644 index 0000000..cc64e8e --- /dev/null +++ b/examples/epigenomics/agents/split_agent.yaml @@ -0,0 +1,12 @@ +agent: + name: SplitAgent + functions: + - name: split + description: Split input_size bytes of data into num_chunks equal chunks. + arguments: + - name: input_size + type: int + - name: num_chunks + type: int + returns: + type: dict diff --git a/examples/epigenomics/config/global_controller.yaml b/examples/epigenomics/config/global_controller.yaml new file mode 100644 index 0000000..6e8b4b0 --- /dev/null +++ b/examples/epigenomics/config/global_controller.yaml @@ -0,0 +1,76 @@ +# Global Controller Configuration — synthetic Epigenomics DAG, local provider +# Lists all agents and the workflow that Ventis manages. +# +# FilterAgent/MapAgent/SortAgent get 2 replicas each since the workflow fans +# out one call per chunk to them -- exercises multi-replica scheduling on a +# purely local, LLM-free run. + +agents: + - name: SplitAgent + replicas: 1 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/split_agent.py + provider: local + + - name: FilterAgent + replicas: 2 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/filter_agent.py + provider: local + + - name: MapAgent + replicas: 2 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/map_agent.py + provider: local + + - name: SortAgent + replicas: 2 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/sort_agent.py + provider: local + + - name: DedupAgent + replicas: 1 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/dedup_agent.py + provider: local + + - name: IndexAgent + replicas: 1 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/index_agent.py + provider: local + + - name: Workflow + replicas: 1 + type: workflow + redis_port: 6379 + api_port: 8080 + workflow_file: workflow/epigenomics_workflow.py + provider: local + +poll_interval: 5 + +redis: + host: localhost + port: 6379 + db: 0 diff --git a/examples/epigenomics/config/policy.yaml b/examples/epigenomics/config/policy.yaml new file mode 100644 index 0000000..2c91415 --- /dev/null +++ b/examples/epigenomics/config/policy.yaml @@ -0,0 +1,20 @@ +# Policy-Based Routing Rules — synthetic Epigenomics DAG +# Each rule defines a match condition (key-value pairs to check against +# request context) and an access list of allowed services. +# Rules are evaluated most-specific-first (most matching keys wins). +# An empty match ({}) acts as a default fallback. + +rules: + - match: + origin: admin + access: all + + - match: {} + access: + - Workflow + - SplitAgent + - FilterAgent + - MapAgent + - SortAgent + - DedupAgent + - IndexAgent diff --git a/examples/epigenomics/workflow/epigenomics_workflow.py b/examples/epigenomics/workflow/epigenomics_workflow.py new file mode 100644 index 0000000..00bcc2b --- /dev/null +++ b/examples/epigenomics/workflow/epigenomics_workflow.py @@ -0,0 +1,97 @@ +# Epigenomics Workflow +# +# WfCommons-style synthetic Epigenomics DAG for local, LLM-free testing: +# 0. SplitAgent - split input_size bytes into num_chunks chunks (single call) +# 1. FilterAgent - per-chunk contaminant filter (fan-out) +# 2. MapAgent - per-chunk alignment, the heaviest stage (fan-out) +# 3. SortAgent - per-chunk sort (fan-out) +# 4. DedupAgent - merge + dedup every sorted chunk (fan-in barrier) +# 5. IndexAgent - build the final index from the merged digest (single call) +# +# Every stage does deterministic SHA-256 work sized off chunk byte counts -- +# no LLM calls, no external services -- so results are reproducible and the +# fan-out width scales with num_chunks. +# +# After running `ventis build` and `ventis deploy`: +# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' \ +# -d '{"input_size": 65536, "num_chunks": 4}' +# curl http://localhost:8080/status/ + +import sys +import os + +# These path inserts are needed when running inside a Docker container +# where all files are copied flat into /app/. +sys.path.insert(0, os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) + +import json + +from deploy import deploy +from agents.split_agent import SplitAgent +from agents.filter_agent import FilterAgent +from agents.map_agent import MapAgent +from agents.sort_agent import SortAgent +from agents.dedup_agent import DedupAgent +from agents.index_agent import IndexAgent + + +def main(input_size: int = 65536, num_chunks: int = 4): + split_agent = SplitAgent() + filter_agent = FilterAgent() + map_agent = MapAgent() + sort_agent = SortAgent() + dedup_agent = DedupAgent() + index_agent = IndexAgent() + + # Stage 0: single call, produces the chunk list the fan-out below runs over. + split = json.loads( + split_agent.split(input_size=input_size, num_chunks=num_chunks).value() + ) + chunks = split["chunks"] + + # Stage 1: fan out one filter call per chunk -- every call returns a Future + # immediately, so all chunks are dispatched before we block on any of them. + filter_futures = { + c["chunk_id"]: filter_agent.filter_contams(chunk_id=c["chunk_id"], size=c["size"]) + for c in chunks + } + filtered = {cid: json.loads(f.value()) for cid, f in filter_futures.items()} + + # Stage 2: fan out alignment -- the heaviest stage -- one call per chunk. + map_futures = { + cid: map_agent.align(chunk_id=cid, size=r["size"], digest=r["digest"]) + for cid, r in filtered.items() + } + mapped = {cid: json.loads(f.value()) for cid, f in map_futures.items()} + + # Stage 3: fan out sort, one call per chunk. + sort_futures = { + cid: sort_agent.sort(chunk_id=cid, size=r["size"], digest=r["digest"]) + for cid, r in mapped.items() + } + sorted_chunks = {cid: json.loads(f.value()) for cid, f in sort_futures.items()} + + # Stage 4: fan-in barrier -- dedup needs every sorted chunk before it can run. + merged = json.loads( + dedup_agent.merge_dedup(chunks=list(sorted_chunks.values())).value() + ) + + # Stage 5: build the final index from the merged digest. + index = json.loads( + index_agent.build_index( + merged_digest=merged["merged_digest"], total_size=merged["total_size"] + ).value() + ) + + return { + "input_size": input_size, + "num_chunks": num_chunks, + "merged_digest": merged["merged_digest"], + "n_chunks": merged["n_chunks"], + "index_digest": index["index_digest"], + } + + +deploy(main, port=8080) diff --git a/examples/helloworld/config/global_controller.yaml b/examples/helloworld/config/global_controller.yaml index 5f6c0cc..0b9c194 100644 --- a/examples/helloworld/config/global_controller.yaml +++ b/examples/helloworld/config/global_controller.yaml @@ -10,7 +10,7 @@ agents: cpu: 1 memory: 512 entrypoint: agents/example_agent.py - provider: local + provider: EC2 - name: VllmAgent replicas: 1 @@ -19,7 +19,7 @@ agents: cpu: 2 memory: 2048 entrypoint: agents/vllm_agent.py - provider: local + provider: EC2 instance_type: t3.micro - name: Workflow @@ -28,7 +28,7 @@ agents: redis_port: 6379 api_port: 8080 # Only needed for workflows, defaults to 8080 if not filled workflow_file: workflow/example_workflow.py - provider: local + provider: EC2 instance_type: t3.micro poll_interval: 5 diff --git a/examples/joke_writer/.car/app/.env.example b/examples/joke_writer/.car/app/.env.example new file mode 100644 index 0000000..b846149 --- /dev/null +++ b/examples/joke_writer/.car/app/.env.example @@ -0,0 +1,20 @@ +# Copy this to `.env` and fill in the token. `config/global_controller.yaml` +# points `env_file:` at that copy, and it reaches every container as +# `docker run --env-file`. +# +# Keep the real token out of THIS file. `.env.example` is the one exception to +# the build context's exclusion of `.env*`, so whatever is written here is baked +# into the image; `.env` itself never enters the build and never leaves the host. + +# A Bedrock API key -- the long-term kind generated in the console, or a +# short-term one. botocore matches this exact name against bedrock-runtime's +# signingName (`bedrock`) and switches the client from SigV4 to bearer auth by +# itself, which is why neither joke_writer.py nor ventis/llm/bedrock.py mentions +# it. An IAM access key works too: drop AWS_ACCESS_KEY_ID and +# AWS_SECRET_ACCESS_KEY in instead and the same client signs with SigV4. +AWS_BEARER_TOKEN_BEDROCK= + +# Neither is a secret, and both have defaults in joke_writer.py -- they are here +# to name what the source reads. +BEDROCK_MODEL_ID=meta.llama3-8b-instruct-v1:0 +AWS_REGION=us-east-1 diff --git a/examples/joke_writer/.car/app/LICENSE b/examples/joke_writer/.car/app/LICENSE new file mode 100644 index 0000000..5600729 --- /dev/null +++ b/examples/joke_writer/.car/app/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 LangChain, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/examples/joke_writer/.car/app/README.md b/examples/joke_writer/.car/app/README.md new file mode 100644 index 0000000..930a410 --- /dev/null +++ b/examples/joke_writer/.car/app/README.md @@ -0,0 +1,177 @@ +# Joke Writer + +A LangGraph map-reduce, ported to Ventis. Derived from +[langchain-ai/langchain-academy](https://github.com/langchain-ai/langchain-academy) +at `fa15bec` (`module-4/studio/map_reduce.py`, MIT — see `LICENSE`). + +Unlike the other targets in `examples/`, **the source here is not unmodified**. +`map_reduce.py` built a `ChatOpenAI` at module scope, and at the time of the port +an agent container had no way to carry an `OPENAI_API_KEY` — the port was blocked +at the credential wall until the model call was rewritten onto Bedrock. That wall +is gone now: `env_file` puts any key in the container. The Bedrock rewrite stayed +anyway, and [What the port cost](#what-the-port-cost) is honest about what that +means. + +## Overview + +Given a topic, the graph splits it into sub-topics, writes one joke per +sub-topic in parallel, then picks the best of them. + +1. `generate_topics` — one LLM call, turns the topic into three sub-topics, + validated into `Subjects`. +2. `generate_joke` — one LLM call per sub-topic. `continue_to_jokes` emits a + `Send` per subject, so this node runs N times per request with no shared + state between the runs. `jokes` is an `Annotated[list, operator.add]`, which + is how the N results merge back into one state. +3. `best_joke` — one LLM call over every joke, returns the winner by index. + +``` + START + | + generate_topics 1 call + | + continue_to_jokes Send x N + / | \ + joke joke joke N calls, no shared state + \ | / + best_joke 1 call + | + END +``` + +### Why this one + +It is the smallest project in reach whose control flow does something a single +process cannot: `Send` fans out to N independent calls per request. Everything +else about it is deliberately boring — four packages, no tools, no external +service, one API key. + +## The port + +| File | What it holds | +| --- | --- | +| `joke_writer.py` | The source. Three prompts, two schemas, three nodes, and the graph — still compiled, never executed under Ventis. | +| `agents/joke_agent.py` | `JokeAgent`. Three methods, each calling the source's node with the node's own state dict. Imports `joke_writer`; restates nothing. | +| `agents/joke_agent.yaml` | The three nodes declared as three functions on one agent. | +| `workflow/joke_workflow.py` | Where the graph went — the edges, the `Send` fan-out and the `operator.add` reducer, re-expressed as ordinary Python. | +| `config/global_controller.yaml` | `JokeAgent` at `replicas: 3`, plus the workflow. | +| `config/policy.yaml` | Default-allow for the two services. Not optional — a missing file kills `ventis deploy`. | + +Two decisions worth naming: + +**One agent, not three.** `generate_topics` and `best_joke` run once per request +and have no resource profile of their own. Splitting them out would buy two more +images and two more Redis round trips. What is hoisted is the fan-out, and that +is a workflow concern. + +**The graph is not the port.** `StateGraph`, `Send` and the `Annotated[list, +operator.add]` reducer are control flow owned by the LangGraph runtime, and +Ventis has no runtime to execute them. The workflow dispatches N +`generate_joke` calls across the three replicas and concatenates the results +itself. Every call is dispatched before any is resolved — `.value()` blocks, so +fusing the two lines into one comprehension would silently serialize the fan-out +and remove the reason to be on Ventis at all. + +## What the port cost + +This is no longer upstream's model stack. `ChatOpenAI` and +`with_structured_output` are gone; `ventis.llm.bedrock.call_bedrock` is the raw +converse API, so each node asks for JSON in its prompt and validates the reply +through the same pydantic schema upstream used. `_extract_json` exists only +because `with_structured_output` used to do that work. + +That rewrite is not something the `porting-to-ventis` skill should do on a +user's project — it is the credential wall, and the skill's instruction is to +report it. It was done here deliberately, so that this example is one that +actually deploys. + +**It would not be necessary today.** The rewrite bought one thing: boto3 builds +no client at import, so the agent could be *loaded* with no secret in the +container, back when `_launch_locally` passed five `-e` flags and all five were +`VENTIS_*`. `env_file` removes that constraint — an `OPENAI_API_KEY` now reaches +a container as readily as a Bedrock one, and upstream's `ChatOpenAI` at module +scope would import fine. What the rewrite still buys is narrower: a module-scope +client turns a missing key into `"No agent loaded"`, while a per-call one turns +it into a real error on `/status`. Worth knowing, not worth a rewrite. + +The example stays on Bedrock because it is the model call that has been end-to-end +verified here, and because `ventis/llm/bedrock.py` is where Ventis writes per-call +token telemetry onto the future. + +## Running it + +Copy `.env.example` to `.env` and put a Bedrock API key in it: + +```shell +cp .env.example .env +$EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... +``` + +`config/global_controller.yaml` points `env_file:` at that file, and every +container gets it as `docker run --env-file`. Nothing in this project reads the +variable: botocore matches the name against `bedrock-runtime`'s signingName and +switches the client from SigV4 to bearer auth on its own, so +`ventis/llm/bedrock.py` still builds a plain `boto3.client("bedrock-runtime")`. +An IAM access key instead of the bearer token works the same way. + +`.env` is gitignored and excluded from the build context — the key is in the +container's environment and not in the image. Deploy checks the path before it +launches anything, so a missing `.env` is one error line rather than three +replicas that come up and fail every request. + +```shell +ventis build +ventis deploy +``` + +```shell +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query": "animals"}' +curl http://localhost:8080/status/ +``` + +```json +{"request_id": "cb6cb62d...", "status": "done", "result": { + "topic": "animals", + "subjects": ["Wildlife Conservation", "Animal Behavior", "Endangered Species"], + "jokes": ["...", "...", "..."], + "best_selected_joke": "Why did the chimpanzee go to the doctor? Because it was going bananas!" +}} +``` + +`BEDROCK_MODEL_ID` and `AWS_REGION` (see `.env.example`) have defaults in +`joke_writer.py`; neither is a secret. The region has to match the one the key +was issued for. + +### Running the source outside Ventis + +`joke_writer.py` imports `ventis.llm.bedrock` first and falls back to the flat +`bedrock` copy an agent image gets, so the compiled graph still runs on its own +from a checkout of this repo: + +```shell +pip install -e ../.. # the ventis package +pip install langgraph pydantic typing_extensions boto3 +``` + +```python +from joke_writer import graph + +graph.invoke({"topic": "animals"}) +``` + +## Provenance + +Taken from `module-4/studio/`, which holds four unrelated graphs sharing one +directory. Only `map_reduce.py` and its license are here. + +| Left behind | Why | +| --- | --- | +| `parallelization.py`, `research_assistant.py`, `sub_graphs.py` | Other graphs in the same studio directory. The first two also need a Tavily key and Wikipedia. | +| `langgraph.json` | Registers all four graphs and points at `./.env`; a trimmed copy would only be useful for LangGraph Studio. | +| The module-4 notebooks | Teaching material for the same code. | +| `OPENAI_API_KEY`, `TAVILY_API_KEY` in `.env.example` | The first belongs to a model call that is no longer here; the second to the two graphs that are not here. | + +Nothing was added at the project root: there is no `pyproject.toml`, `setup.py` +or `requirements.txt`, exactly as upstream has none for module-4. That is why +`config/global_controller.yaml` has to declare `requirements:` by hand. diff --git a/examples/joke_writer/.car/app/joke_workflow.py b/examples/joke_writer/.car/app/joke_workflow.py new file mode 100644 index 0000000..76e4027 --- /dev/null +++ b/examples/joke_writer/.car/app/joke_workflow.py @@ -0,0 +1,39 @@ +r"""CanyonOS Core workflow for the map-reduce joke writer. + +This file is where the graph went. `generate_topics -> continue_to_jokes -> +generate_joke x N -> best_joke` is not a compiled StateGraph any more; it is the +three statements below, and the `Send` fan-out is N calls dispatched across +JokeAgent's replicas. + + curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query": "animals"}' + curl http://localhost:8080/status/ +""" + +import json + +from deploy import deploy +from joke_writer import JokeAgent + + +def main(query): + """Route: POST /main {"query": ""}""" + agent = JokeAgent() + + subjects = json.loads(agent.generate_topics(topic=query).value())["subjects"] + + futures = [agent.generate_joke(subject=s) for s in subjects] + written = [json.loads(f.value()) for f in futures] + written_jokes = [joke for result in written for joke in result["jokes"]] + + best = json.loads(agent.best_joke(topic=query, jokes=written_jokes).value()) + + return { + "topic": query, + "subjects": subjects, + "jokes": written_jokes, + "best_selected_joke": best["best_selected_joke"], + } + + +deploy(main, port=8080) diff --git a/examples/joke_writer/.car/app/joke_writer.py b/examples/joke_writer/.car/app/joke_writer.py new file mode 100644 index 0000000..9f96eb5 --- /dev/null +++ b/examples/joke_writer/.car/app/joke_writer.py @@ -0,0 +1,164 @@ +"""Map-reduce joke writer. + +Derived from langchain-ai/langchain-academy `module-4/studio/map_reduce.py` +(MIT, see LICENSE). The graph shape, the three prompts and the two schemas are +upstream's. The model call is not: upstream builds a `ChatOpenAI` at module +scope, and when this was ported nothing could carry an OPENAI_API_KEY into an +agent container. Bedrock reaches the model through boto3, which builds no client +at import, so the same code loaded with no secret injected. + +`env_file` has since removed that constraint -- the key now travels to the +container in a .env and botocore reads AWS_BEARER_TOKEN_BEDROCK on its own. The +rewrite stayed regardless; README.md says what that costs. + +`with_structured_output` went with it. `call_bedrock` is the raw converse API, so +each node asks for JSON in the prompt and validates the reply through the same +pydantic schema upstream used. +""" + +import json +import operator +import os +import re +from typing import Annotated + +from typing_extensions import TypedDict + +from pydantic import BaseModel, ValidationError + +from langgraph.constants import Send +from langgraph.graph import END, StateGraph, START + +# Ventis copies bedrock.py flat into every agent image; the package path is for +# running this module outside a container. +try: + from ventis.llm.bedrock import call_bedrock +except ImportError: + from bedrock import call_bedrock + +# Prompts we will use. Upstream's, plus the JSON instruction that +# `with_structured_output` used to add on our behalf. +subjects_prompt = """Generate a list of 3 sub-topics that are all related to this overall topic: {topic}. +Respond with JSON only, no prose: {{"subjects": ["...", "...", "..."]}}""" +joke_prompt = """Generate a joke about {subject}. +Respond with JSON only, no prose: {{"joke": "..."}}""" +best_joke_prompt = """Below are a bunch of jokes about {topic}. Select the best one! Return the ID of the best one, starting 0 as the ID for the first joke. Jokes: \n\n {jokes} +Respond with JSON only, no prose: {{"id": 0}}""" + +# LLM. Both are read once at import; the container gets them from its +# environment, and neither is a secret. +MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0") +REGION = os.environ.get("AWS_REGION", "us-east-1") + + +def _extract_json(text): + """Pull the first JSON object out of a model reply. + + Even told to answer with JSON only, a model wraps it in a ```json fence or + prefaces it with a sentence. Upstream never needed this because + `with_structured_output` handled it; the converse API does not. + """ + text = re.sub(r"^\s*```(?:json)?|```\s*$", "", text.strip(), flags=re.MULTILINE) + try: + return json.loads(text) + except json.JSONDecodeError: + pass + # Fall back to the outermost braced span. + match = re.search(r"\{.*\}", text, flags=re.DOTALL) + if not match: + raise ValueError(f"joke_writer: no JSON in model output: {text!r}") + return json.loads(match.group(0)) + + +def _ask(prompt, schema, max_tokens): + """One converse() call, validated into `schema`. + + Raising on a bad reply is deliberate. A node that returned a default would + put a plausible-looking wrong answer into the state, and the reduce step + downstream indexes into the jokes list by an id the model chose -- a silent + default there picks the wrong joke instead of failing. + """ + response = call_bedrock( + model_id=MODEL_ID, + messages=[{"role": "user", "content": [{"text": prompt}]}], + inference_config={"maxTokens": max_tokens, "temperature": 0.0}, + region=REGION, + ) + text = response["output"]["message"]["content"][0]["text"] + if not text: + raise ValueError("joke_writer: LLM returned no output.") + try: + return schema(**_extract_json(text)) + except (ValidationError, TypeError) as exc: + raise ValueError( + f"joke_writer: {schema.__name__} not satisfied by model output: {text!r}" + ) from exc + + +# Define the state +class Subjects(BaseModel): + subjects: list[str] + +class BestJoke(BaseModel): + id: int + +class OverallState(TypedDict): + topic: str + subjects: list + jokes: Annotated[list, operator.add] + best_selected_joke: str + +def generate_topics(state: OverallState): + prompt = subjects_prompt.format(topic=state["topic"]) + response = _ask(prompt, Subjects, max_tokens=300) + return {"subjects": response.subjects} + +class JokeState(TypedDict): + subject: str + +class Joke(BaseModel): + joke: str + +def generate_joke(state: JokeState): + prompt = joke_prompt.format(subject=state["subject"]) + response = _ask(prompt, Joke, max_tokens=300) + return {"jokes": [response.joke]} + +def best_joke(state: OverallState): + jokes = "\n\n".join(state["jokes"]) + prompt = best_joke_prompt.format(topic=state["topic"], jokes=jokes) + response = _ask(prompt, BestJoke, max_tokens=100) + if not 0 <= response.id < len(state["jokes"]): + raise ValueError( + f"joke_writer: model chose joke {response.id} of {len(state['jokes'])}." + ) + return {"best_selected_joke": state["jokes"][response.id]} + +def continue_to_jokes(state: OverallState): + return [Send("generate_joke", {"subject": s}) for s in state["subjects"]] + +# Construct the graph: here we put everything together to construct our graph +graph_builder = StateGraph(OverallState) +graph_builder.add_node("generate_topics", generate_topics) +graph_builder.add_node("generate_joke", generate_joke) +graph_builder.add_node("best_joke", best_joke) +graph_builder.add_edge(START, "generate_topics") +graph_builder.add_conditional_edges("generate_topics", continue_to_jokes, ["generate_joke"]) +graph_builder.add_edge("generate_joke", "best_joke") +graph_builder.add_edge("best_joke", END) + +# Compile the graph +graph = graph_builder.compile() + + +class JokeAgent(object): + """The graph's nodes, exposed under the class name `agent.name` declares.""" + + def generate_topics(self, topic: str) -> dict: + return generate_topics({"topic": topic}) + + def generate_joke(self, subject: str) -> dict: + return generate_joke({"subject": subject}) + + def best_joke(self, topic: str, jokes: list) -> dict: + return best_joke({"topic": topic, "jokes": jokes}) diff --git a/examples/joke_writer/.car/config/global_controller.yaml b/examples/joke_writer/.car/config/global_controller.yaml new file mode 100644 index 0000000..8ed54ab --- /dev/null +++ b/examples/joke_writer/.car/config/global_controller.yaml @@ -0,0 +1,52 @@ +# Deployment manifest for the map-reduce joke writer. +# +# `entrypoint` is the copied source itself: the adapter is appended to the +# bottom of joke_writer.py, so the module the agent needs is the one the class +# already lives in. + +agents: + - name: JokeAgent + # The fan-out. `generate_joke` is stateless, so the controller picks a + # replica at random per call and the workflow's N dispatched calls spread + # across these three. + entrypoint: joke_writer.py + provider: local + replicas: 3 + redis_port: 6379 + resources: + cpu: 1 + memory: 1024 + requirements: + - langgraph + - pydantic + - typing_extensions + + - name: Workflow + type: workflow + workflow_file: joke_workflow.py + api_port: 8080 + provider: local + replicas: 1 + redis_port: 6379 + +poll_interval: 5 + +redis: + host: localhost + port: 6379 + db: 0 + +otel: + # The dashboard api's own OTLP ingest. Must be the full url including the + # path: the http exporter uses an explicitly-passed endpoint verbatim and + # only appends /v1/traces when reading OTEL_EXPORTER_OTLP_ENDPOINT. + destinations: + - name: local + protocol: http + endpoint: http://host.docker.internal:3000/v1/traces + headers: {} + +# Relative to the application root (the directory `ventis` runs from), not +# `.car`. .env is gitignored and excluded from the build context; +# .env.example names what belongs in it. +env_file: .env diff --git a/examples/joke_writer/.car/config/joke_agent.yaml b/examples/joke_writer/.car/config/joke_agent.yaml new file mode 100644 index 0000000..a5bc13f --- /dev/null +++ b/examples/joke_writer/.car/config/joke_agent.yaml @@ -0,0 +1,35 @@ +# The graph's three nodes, exposed as three methods on one agent. +# +# One agent, not three. `generate_topics` and `best_joke` run once per request +# and have no resource profile of their own; splitting them out would add two +# images, two dependency trees and a Redis round trip to buy nothing. What is +# hoisted is the `Send` fan-out, and that is a workflow concern. + +agent: + name: JokeAgent + functions: + - name: generate_topics + description: Split a topic into three related sub-topics. + arguments: + - name: topic + type: str + returns: + type: dict + + - name: generate_joke + description: Write one joke about one subject. + arguments: + - name: subject + type: str + returns: + type: dict + + - name: best_joke + description: Pick the best joke out of the ones written for a topic. + arguments: + - name: topic + type: str + - name: jokes + type: list + returns: + type: dict diff --git a/examples/joke_writer/README.md b/examples/joke_writer/README.md index 3ba7930..930a410 100644 --- a/examples/joke_writer/README.md +++ b/examples/joke_writer/README.md @@ -80,7 +80,7 @@ converse API, so each node asks for JSON in its prompt and validates the reply through the same pydantic schema upstream used. `_extract_json` exists only because `with_structured_output` used to do that work. -That rewrite is not something the `porting-to-canyonos-core` skill should do on a +That rewrite is not something the `porting-to-ventis` skill should do on a user's project — it is the credential wall, and the skill's instruction is to report it. It was done here deliberately, so that this example is one that actually deploys. @@ -107,12 +107,6 @@ cp .env.example .env $EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... ``` -> **`env_file:` needs PR #53** (`jiajunh/can-232-...`), still open against main. -> Until it merges nothing in `ventis/` reads the key, so the steps below leave -> the container without a credential and every request answers a Bedrock -> credential error. `python ../../.claude/skills/porting-to-canyonos-core/validate.py .` -> reports this as V030 and stops reporting it the day the PR lands. - `config/global_controller.yaml` points `env_file:` at that file, and every container gets it as `docker run --env-file`. Nothing in this project reads the variable: botocore matches the name against `bedrock-runtime`'s signingName and diff --git a/examples/joke_writer/agents/joke_agent.py b/examples/joke_writer/agents/joke_agent.py deleted file mode 100644 index 8fa3b93..0000000 --- a/examples/joke_writer/agents/joke_agent.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Ventis entrypoint for the map-reduce joke writer. - -Nothing here restates the project. The three prompts, the two schemas and the -Bedrock binding all live in `joke_writer.py` and are reached with an import -- -the whole project tree is in the image. - -What could not be reused is the graph itself. `StateGraph`, the `Send` in -`continue_to_jokes` and the `Annotated[list, operator.add]` reducer are control -flow owned by the LangGraph runtime, and Ventis has no runtime to execute them. -That wiring is re-expressed as ordinary Python in workflow/joke_workflow.py, -where the fan-out becomes N dispatched calls across this agent's replicas. The -nodes those edges connected are imported, unchanged. - -The module is imported whole rather than by name so that -`joke_writer.generate_joke` inside a method named `generate_joke` reads as what -it is: the source's node. -""" - -# The source tree. Importing it reads BEDROCK_MODEL_ID and AWS_REGION, imports -# bedrock.py (which builds a RedisClient at module scope) and compiles the graph -# -- but it constructs no API client, so the import needs no credential. -# -# The credential arrives by a different road: `env_file` in -# config/global_controller.yaml hands the container a .env holding -# AWS_BEARER_TOKEN_BEDROCK, and botocore picks that name up by itself. Nothing -# here or in joke_writer.py names it. -# -# Constructing no client at import is no longer what makes this agent loadable -- -# env_file would carry a key to a module-scope client too. It only changes the -# failure: a missing key is an error on /status rather than "No agent loaded". -import joke_writer - - -class JokeAgent(object): - """The graph's nodes, exposed under the class name `agent.name` declares.""" - - # No constructor arguments -- LocalController does `JokeAgent()`. The model - # id and region are the source's own module-level constants, read from the - # environment there; there is nothing to configure here. - - def generate_topics(self, topic: str) -> dict: - """Split a topic into sub-topics. Returns {"subjects": [...]}. - - Synchronous by signature -- the executor calls this with no `await`, and - returning a coroutine would put `` into Redis. - """ - # The node's own state dict goes in, the node's own return comes out. - # Both hold nothing but str and list, so the executor's json.dumps is - # happy without a serializer -- unlike a graph that hands back messages. - return joke_writer.generate_topics({"topic": topic}) - - def generate_joke(self, subject: str) -> dict: - """Write one joke about one subject. Returns {"jokes": ["..."]}. - - The single-element list is the node's own shape: it is what - `Annotated[list, operator.add]` merged N of. The workflow does that - concatenation now. - """ - return joke_writer.generate_joke({"subject": subject}) - - def best_joke(self, topic: str, jokes: list) -> dict: - """Pick the winner. Returns {"best_selected_joke": "..."}.""" - return joke_writer.best_joke({"topic": topic, "jokes": jokes}) diff --git a/examples/joke_writer/agents/joke_agent.yaml b/examples/joke_writer/agents/joke_agent.yaml deleted file mode 100644 index fee8607..0000000 --- a/examples/joke_writer/agents/joke_agent.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# The graph's three nodes, exposed as three methods on one agent. -# -# One agent, not three. `generate_topics` and `best_joke` run once per request -# and have no resource profile of their own, so splitting them out would add -# two images, two dependency trees and a Redis round trip to buy nothing. What -# is hoisted is the `Send` fan-out, and that is a workflow concern, not a -# second agent: the workflow dispatches N `generate_joke` calls and the -# routing table spreads them across this agent's replicas. -# -# This file's basename names the generated stub, not the agent. Sharing it with -# joke_agent.py is why both land at /app/joke_agent.py -- the entrypoint is -# copied last and wins it, so the agent container loads the real class while the -# stub keeps /app/agents/joke_agent.py for callers. What the basename must not -# match is a source module: a `joke_writer.yaml` would put a stub at -# /app/joke_writer.py, on top of the module the adapter imports. - -agent: - name: JokeAgent - functions: - # Node 1 of the graph. One LLM call, structured output into `Subjects`. - - name: generate_topics - description: Split a topic into three related sub-topics. - arguments: - # Must equal the Python parameter name character for character -- - # LocalController calls method(**args). - - name: topic - type: str - # dict -> the workflow must json.loads what .value() hands back - returns: - type: dict - - # Node 2. The fan-out: one call per sub-topic, no shared state between - # them. This is the only reason this project is on Ventis. - - name: generate_joke - description: Write one joke about one subject. - arguments: - - name: subject - type: str - returns: - type: dict - - # Node 3. The reduce: one call over every joke the fan-out produced. - - name: best_joke - description: Pick the best joke out of the ones written for a topic. - arguments: - - name: topic - type: str - # `list` is a builtin, so the stub's annotation resolves. `list[str]` - # would be pasted into the AST verbatim and NameError on import. - - name: jokes - type: list - returns: - type: dict diff --git a/examples/joke_writer/config/global_controller.yaml b/examples/joke_writer/config/global_controller.yaml deleted file mode 100644 index eb26090..0000000 --- a/examples/joke_writer/config/global_controller.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Deployment manifest for the map-reduce joke writer. -# -# `entrypoint` is the adapter, which imports the untouched-in-shape source tree. -# -# The source has no pyproject.toml, setup.py or setup.cfg, so the Dockerfile's -# `-e .` is skipped -- silently. It does not matter here: `joke_writer.py` sits -# at the project root, so it lands flat at /app, which is sys.path[0]. A source -# laid out under src/ would need its own packaging metadata to import at all. - -agents: - - name: JokeAgent - # The fan-out. `generate_joke` is stateless, so LocalController picks a - # replica at random per call and the workflow's N dispatched calls spread - # across these three. N is whatever the model returns (the prompt asks for - # three sub-topics); replicas bound how many run at once, not how many run. - replicas: 3 - redis_port: 6379 - resources: - cpu: 1 - memory: 1024 - entrypoint: agents/joke_agent.py - provider: local - # What the source imports beyond the runtime's own list, which the generator - # prepends. boto3 is already in it, which is the whole reason the Bedrock - # call needs nothing declared here. The graph is never executed in this - # container, but `joke_writer.py` imports langgraph at module scope, so it - # still has to be installed. - requirements: - - langgraph - - pydantic - - typing_extensions - - - name: Workflow - type: workflow - replicas: 1 - redis_port: 6379 - api_port: 8080 - workflow_file: workflow/joke_workflow.py - provider: local - -poll_interval: 5 - -redis: - host: localhost - port: 6379 - db: 0 - -# `provider` must be lowercase. InstanceManager.launch_all tests -# `provider == "local"` to decide whether to reserve a host port; `Local` fails -# that test, reserved_port stays None, and Local/_runtime.py raises -# `int() argument must be ... not 'NoneType'` before any container starts. -# -# The credential. `_launch_locally` passes exactly five `-e` flags, all VENTIS_*, -# and .env is excluded from the build context, so for a while the only model call -# that could work here was one that needed no secret in the container: boto3 -# resolving an instance role per call. `env_file` is what changed. It points at a -# local .env, unresolved paths relative to this project root, and every container -# gets it as `docker run --env-file` -- so the key is in the environment without -# ever entering the image. -# -# What lands there is AWS_BEARER_TOKEN_BEDROCK. Nothing in this project reads it: -# botocore matches the name against bedrock-runtime's signingName and switches -# the client to bearer auth on its own, so `ventis/llm/bedrock.py` still builds a -# plain `boto3.client("bedrock-runtime")`. -# -# Deploy fails here rather than in a container: resolve_env_file checks the path -# before InstanceManager launches anything, so a missing .env is one error line -# instead of three replicas that come up and then answer -# {"status": "error", "error": "Unable to locate credentials"} on every request. -# -# What it costs: this is no longer upstream's model stack. See README.md. - -# Relative to this project root, same as `entrypoint` and `workflow_file`. -# .env is gitignored and excluded from the build context; .env.example names -# what belongs in it. -# -# NOTE: this key needs PR #53 (jiajunh/can-232-...), which is still open against -# main. On main nothing reads it -- `grep -rn env_file ventis/` finds no hits -- -# so the key is inert, no credential reaches the container, and every request -# answers a Bedrock credential error. `validate.py` reports that as V030 until -# the PR lands. -env_file: .env diff --git a/examples/joke_writer/config/policy.yaml b/examples/joke_writer/config/policy.yaml deleted file mode 100644 index 2cb9cb3..0000000 --- a/examples/joke_writer/config/policy.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Policy-Based Routing Rules — map-reduce joke writer -# Each rule defines a match condition (key-value pairs checked against the -# request context) and an access list of allowed services. -# Rules are evaluated most-specific-first (most matching keys wins). -# An empty match ({}) acts as a default fallback. -# -# This file IS optional -- `_load_policy_rules` logs "No policy file found" and -# returns [], and `_check_policy` allows everything when the rule list is empty. -# What is not safe is a half-written one: past the isfile() guard the read is -# unguarded, so an empty file (`.get("rules")` on None) or a null `rules:` -# (`None.sort()`) raises inside GlobalController.__init__ and `ventis deploy` -# dies before any container starts. Delete it or fill it; do not leave it empty. - -rules: - # Default fallback: the workflow and the one agent behind it. A service left - # out of this list answers "Unauthorized: Policy denied access to service". - - match: {} - access: - - Workflow - - JokeAgent diff --git a/examples/joke_writer/workflow/joke_workflow.py b/examples/joke_writer/workflow/joke_workflow.py deleted file mode 100644 index 9fdf3cc..0000000 --- a/examples/joke_writer/workflow/joke_workflow.py +++ /dev/null @@ -1,59 +0,0 @@ -r"""Ventis workflow for the map-reduce joke writer. - -This file is where the graph went. `generate_topics -> continue_to_jokes -> -generate_joke x N -> best_joke` is not a compiled StateGraph any more; it is the -three statements below, and the `Send` fan-out is N calls dispatched across -JokeAgent's replicas. - -The function is `main` and its one argument is `query` because the deployment -platform's test endpoint posts to a hardcoded /main with a strictly validated -{query: string} body. Ventis would serve any name and any kwargs -- the route is -the function's __name__ and the body is splatted in -- so nothing here fails if -you rename it; it just stops being reachable through the platform. - - curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query": "animals"}' - curl http://localhost:8080/status/ -""" - -import json - -from deploy import deploy -from agents.joke_agent import JokeAgent - - -def main(query): - """Route: POST /main {"query": ""}""" - agent = JokeAgent() - - # Node 1: one call, and the fan-out width comes out of it. The agent's own - # parameter is still `topic` -- that name is bound by joke_agent.yaml and the - # source's node, and only the workflow's entry point is pinned to `query`. - subjects = json.loads(agent.generate_topics(topic=query).value())["subjects"] - - # `continue_to_jokes`, re-expressed. Every call is dispatched before any - # of them is resolved -- .value() blocks, so fusing these two lines into one - # comprehension would run the jokes one after another. It would not error; - # the fan-out would just be gone, and with it the reason to be on Ventis. - futures = [agent.generate_joke(subject=s) for s in subjects] - written = [json.loads(f.value()) for f in futures] - - # `Annotated[list, operator.add]`, re-expressed: the reducer that merged N - # single-joke lists back into one list was part of the graph, not of a node. - written_jokes = [joke for result in written for joke in result["jokes"]] - - # Node 3: the reduce. `list` in the yaml is what lets this argument through. - best = json.loads(agent.best_joke(topic=query, jokes=written_jokes).value()) - - return { - "topic": query, - "subjects": subjects, - "jokes": written_jokes, - "best_selected_joke": best["best_selected_joke"], - } - - -# This file is exec'd, not imported, so __name__ == "__main__" here and any -# `if __name__ == "__main__":` block would run in production. deploy() blocks -# on app.run(); nothing after it executes. -deploy(main, port=8080) diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md new file mode 100644 index 0000000..fe6dd49 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md @@ -0,0 +1,240 @@ +--- +name: porting-to-canyonos-core +description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. +compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. +--- + +# Port an agent project to CanyonOS Core + +CanyonOS Core is the product name. Its compatibility executable and Python +package remain `ventis`; environment variables and Docker resources retain the +`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding +strings. Do not rename them. + +## Load references only when needed + +- Read [references/packaging.md](references/packaging.md) when a source import + does not resolve from `/app`, the source is nested, or packaging metadata is + involved. +- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target + includes `llm_proxy`. +- Read [references/ec2.md](references/ec2.md) only when any config entry uses + `provider: EC2`. +- Read [references/troubleshooting.md](references/troubleshooting.md) after a + failed build, image probe, deploy, or request. +- Read [references/runtime-contract.md](references/runtime-contract.md) when a + validator finding needs explanation or the runtime mechanism is unclear. + +## Goal: thin scaffolding beside untouched source + +```text +agents/.yaml one callable surface per service +agents/.py one thin adapter per service, when needed +workflow/_workflow.py HTTP entry point; calls deploy() +config/global_controller.yaml deployment manifest +config/policy.yaml optional access restriction +pyproject.toml conditional nested-import scaffolding + unchanged +``` + +The file count follows the deployment. A multi-agent port has one yaml/adapter +pair per service that is worth deploying separately. If a source class already +satisfies the runtime contract, point its config entry at that file and do not +copy it into an adapter. + +Everything the source already owns—prompts, tools, schemas, parsing, retries, +model clients, and node bodies—is imported. The port re-expresses only the +CanyonOS Core boundary and framework-owned orchestration. + +The port root is the existing repository root and the directory from which +`ventis build` runs. Write scaffolding there beside existing directories. If the +repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, +and `config/` beside it. Never move or copy the repository into a new `src/` +directory, and never create an outer wrapper merely for the port. + +## 1. Survey before writing + +Identify: + +1. The source entry point and callable input/output. +2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, + `Send`, `Command`, interrupts). +3. Runtime-injected services nodes read: stores, context, memory, sessions, or + callback managers. +4. Sync versus async boundaries. +5. Imports and declared runtime dependencies. +6. Model provider, credential names, streaming use, and optional `llm_proxy`. +7. Whether independent work fans out and benefits from separate replicas. +8. Whether source imports resolve from the project root that becomes `/app`. + +Run the validator once now. Its header detects capabilities directly from the +importable runtime rather than external development metadata: + +```bash +python /validate.py . +``` + +If config or agent yaml is malformed, the validator defers to `ventis build`. +Capability-gated findings say which runtime behavior is available. + +## 2. Choose service boundaries + +Start with one service. Split only when it creates independent parallel work or +a distinct resource/replica profile. + +- Keep a ReAct loop together; every turn needs shared message history. +- Hoist supervisor task lists and `Send`-style fan-out into the workflow. +- Do not create a one-replica service with no distinct resource profile merely + to mirror every source graph node. + +Rewrite framework-owned edges as ordinary Python. Import the connected node +functions unchanged. Construct runtime-injected service objects from source +configuration; do not invent models, dimensions, stores, or defaults silently. +Report any choice the source does not specify. + +## 3. Write declarations and adapters + +### Agent yaml + +Use one yaml per deployed service. Argument types are bare builtins only: +`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is +required by the generated stub. `returns.type` is documentation; use `dict` or +`list` to signal that workflow callers must `json.loads` the returned string. + +### Adapter + +The entrypoint exposes a module-level class named exactly `agent.name`. It +constructs with no arguments and its declared methods are synchronous. Read +configuration from the environment in `__init__`. Bridge source coroutines +inside a synchronous method with `asyncio.run(...)`. Serialize framework objects +with their own JSON-safe serializer before returning. + +Do not duplicate source prompts, tools, schemas, or model calls. Keep the source +provider and SDK. + +### Workflow + +Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. +Import generated stubs by yaml basename and agent class name: + +```python +from deploy import deploy +from agents. import +``` + +The deployment platform sends `{query: string}` to `/main`. Pack richer input +inside `query`; any additional workflow parameter has a default. + +Dispatch every remote call before resolving any future: + +```python +futures = [agent.work(item=item) for item in items] +results = [json.loads(future.value()) for future in futures] +``` + +Do not fuse dispatch and `.value()` in one comprehension; that silently +serializes fan-out. Do not add an `if __name__ == "__main__":` block: the +workflow is executed with `__name__ == "__main__"` in production. + +### Config + +For each service, keep these names aligned: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a +list of distribution-name strings. Put `env_file` at config top level when the +runtime capability is available. Omit `policy.yaml` unless access must be +restricted; if present, give it a non-empty `rules` list. + +## Hard rules + +Capitalized **MUST** and **NEVER** are reserved for port-breaking or +source-integrity rules. The owner column states where each is decided. + +| ID | Rule | Owner | +|---|---|---| +| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | +| M2 | The class MUST construct with no arguments | V007 | +| M3 | yaml argument names MUST match Python parameter names | V008 | +| M4 | yaml argument types MUST be bare builtins | V010 | +| M5 | Declared adapter methods MUST be synchronous | V009 | +| M6 | Config names MUST match yaml agent names | build | +| M7 | Config names MUST not collide after lowercase normalization | build output | +| M8 | Local provider MUST be lowercase `local` | deploy preflight | +| M9 | `replicas` MUST be an integer | deploy preflight | +| M10 | `requirements` MUST be a list of strings | build | +| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | +| M12 | Workflow MUST NEVER contain a main guard | V017 | +| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | +| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | +| M15 | Workflow MUST import stubs from `agents.` | V023 | +| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | +| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | +| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | +| M19 | NEVER hardcode or bake a real credential into an image | W003 | +| M20 | NEVER edit or vendor the source tree | `git status` | +| M21 | NEVER swap the source LLM provider | review | +| M22 | NEVER silently move, drop, or reclassify source dependencies | review | +| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | +| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | + +## 4. Validate, build, and probe + +Run static preflight, then let the build own build-time validation: + +```bash +python /validate.py . +ventis build -c config/global_controller.yaml +``` + +A green build never imports the adapter. Probe each agent image in this order: + +```bash +# Runtime startup path + docker run --rm ventis- \ + python -c "import local_controller" + +# Agent load path; include --env-file when configured + docker run --rm --env-file ventis- \ + python -c "import importlib.util,sys; \ +s=importlib.util.spec_from_file_location('m','.py'); \ +m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ +m.();print('ok')" +``` + +Also probe the workflow image with `python -c "import local_controller"`; it has +its own dependency resolve and generated-stub imports. + +Then deploy, send a representative request, and poll its status: + +```bash +ventis deploy -c config/global_controller.yaml +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query":""}' +curl http://localhost:8080/status/ +``` + +A successful outer request with a source-level failure still proves the port +reached and returned the source behavior. Record the distinction. + +## 5. Clean up + +After collecting evidence, stop foreground deploy with Ctrl+C and wait for +controller cleanup. Remove exact leftovers if startup crashed. Then remove build +products and exact images from this config: + +```bash +ventis clean +docker image rm ventis- \ + ventis- + +test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container +docker ps -a --format '{{.Names}}' +``` + +`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it +does not remove containers or images. Keep port scaffolding, untouched source, +and requested logs or reports. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md new file mode 100644 index 0000000..e06daa0 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md @@ -0,0 +1,41 @@ +# EC2 deployment + +Read this only when at least one config entry uses `provider: EC2`. + +## Configuration + +Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The +top-level `ec2` block supplies the runtime's required infrastructure and SSH +settings. Read the target checkout's deploy preflight and EC2 runtime before +writing the block; do not copy values from an example environment. + +Typical required categories are: + +- AMI and instance type +- region and subnet +- security groups +- SSH user and credentials accepted by the runtime + +`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof +that provisioning, SSH, image transfer, or remote container startup works. + +## Networking + +A remote container's `host.docker.internal` names its own EC2 Docker host. It +does not name the local controller machine. Databases, model proxies, and other +services must use addresses reachable from every selected host. + +The environment file may be copied temporarily to a remote host by runtimes that +expose the `env_file` capability. Confirm behavior from the capability probe and +target runtime rather than assuming local Docker semantics. + +## Probes and cleanup + +Run the same runtime and adapter probes against the exact image before remote +deployment. After deploy, verify the remote container logs; controller health +can be green even when agent loading failed. + +Stop foreground deploy normally so the controller can terminate recorded EC2 +instances. If provisioning or startup fails before an instance is recorded, +inspect the cloud provider directly and remove exact leaked resources. Never use +a broad cleanup command against unrelated instances. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md new file mode 100644 index 0000000..f726bd7 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md @@ -0,0 +1,64 @@ +# LLM proxy integration + +Read this only when the target checkout contains `llm_proxy` or the deployment +explicitly routes model SDKs through it. + +## Preserve provider protocols + +The proxy redirects provider endpoints; it does not convert providers. Keep the +source SDK, model ID, request body, and response parsing unchanged. + +Configure only the provider variables the source uses: + +```dotenv +OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 +ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic +AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock +``` + +Some SDKs refuse to initialize without caller credentials. Give agent containers +non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS +credentials in the separate proxy process, not in the port's `env_file`. + +## Start locally + +The proxy defaults conflict with a typical deployment: host loopback is not +reachable from a container, and port 8080 is normally used by the workflow API. +Use a non-loopback bind and a different port: + +```bash +PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy +curl http://127.0.0.1:8081/healthz +``` + +Local CanyonOS Core containers resolve `host.docker.internal` through their +Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the +machine running `ventis deploy`. Distributed deployments need a reachable proxy +address or one proxy on each host. + +## Supported call shape + +The implementation buffers complete requests and responses: + +- OpenAI and Anthropic non-streaming HTTP calls are forwarded. +- Bedrock `invoke` is reissued through the proxy's boto3 client. +- OpenAI/Anthropic streaming is unsupported. +- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are + unsupported. + +Survey the source before selecting the proxy. Do not silently disable streaming; +report the unsupported behavior and stop. + +## Credential behavior + +- The OpenAI adapter removes caller authorization and inserts the proxy key. +- The Anthropic adapter removes caller key headers and inserts the proxy key. +- Botocore still signs requests sent to a custom endpoint, so a caller may need + placeholder AWS credentials even though the proxy reissues upstream with its + own identity. +- `/healthz` proves provider registration and Flask availability, not upstream + credential validity. + +OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return +JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed +with the upstream status and are not byte-for-byte passthrough. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md new file mode 100644 index 0000000..8520c4a --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md @@ -0,0 +1,81 @@ +# Packaging and import roots + +Read this reference when an adapter imports nested source code, the source uses a +`src/` layout, or V031 reports an import-root problem. + +## What `/app` can import + +CanyonOS Core preserves project-relative paths in the image and starts Python at +`/app`. Without an editable install, Python resolves names rooted there: + +- `/app/tools.py` as `import tools` +- `/app/pkg/__init__.py` as `import pkg` +- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace + directories without `__init__.py` + +It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become +an import root first. + +## Detect support, do not infer it from release history + +Run: + +```bash +python /validate.py . +``` + +Read the `editable_install` capability. If it is unavailable and the original +import cannot resolve from `/app`, report a runtime capability blocker and stop. +Do not add a `sys.path` hack or relocate source files. + +## Root metadata is the trigger + +When editable install is supported, only packaging metadata at the **port root** +triggers `pip install -e .`: + +```text +port-root/pyproject.toml detected +port-root/source/pyproject.toml ignored as an install trigger +``` + +A nested source repository may remain untouched. Add minimal root scaffolding +that points package discovery at the existing source package: + +```toml +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "canyonos-port" +version = "0.0.0" +dependencies = [] + +[tool.setuptools.packages.find] +where = ["source/src"] +include = ["pkg*"] +namespaces = true +``` + +Set `where` and `include` from the actual tree and original import spelling. Do +not reference a README or license from this wrapper metadata; file sweeps differ +by runtime capability and a missing referenced file makes the image build fail. + +## Dependencies in nested metadata + +A nested `pyproject.toml` is not installed merely because its Python files are +copied. Keep the source declaration unchanged and repeat its runtime +distributions in each relevant config entry's `requirements` list. This is +compatibility scaffolding, not permission to drop, move, or reclassify declared +dependencies. + +If source metadata is already at the port root, do not create a wrapper. Its +project dependencies participate in the same resolver as config requirements. +Report declared-but-unused toolchain dependencies and their image cost; let the +owner decide whether source metadata should change. + +## Validation boundary + +`ventis build` owns packaging syntax and installation errors. `validate.py` +checks only whether adapter imports appear to require a nested root that the +runtime will not expose. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md new file mode 100644 index 0000000..94f7401 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md @@ -0,0 +1,187 @@ +# CanyonOS Core runtime contract + +The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the +CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for +Docker resources. + +Read this reference when implementing an adapter or explaining a validator +finding. Runtime-dependent behavior is expressed as capabilities; run +`validate.py` against the target environment instead of inferring support from +release history. + +## Project root and discovery + +`ventis build` uses the current working directory as the project root. + +| Input | Discovery | +|---|---| +| `agents/*.yaml` | direct yaml glob under the project root | +| `config/global_controller.yaml` | default config, overridable with `-c` | +| workflow | `workflow_file` on a `type: workflow` entry | +| policy | `policy.yaml` beside the selected config file | +| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | + +The config name, yaml `agent.name`, and entrypoint class name form one binding: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +A missing match may skip an image while the command continues, so inspect build +output and generated image tags. + +## Agent yaml and generated stubs + +The consumed yaml shape is: + +```yaml +agent: + name: ExampleAgent + functions: + - name: work + arguments: + - name: query + type: str + returns: + type: dict +``` + +Argument annotations are generated from bare names without adding imports. Use +builtins. Generated methods have no defaults, so every declared argument is +required at the stub call site. `returns` does not control runtime conversion; +it documents whether workflow code should parse the returned string. + +Stub destinations differ by runtime capability and entrypoint layout. Workflow +code in this port convention imports the generated class from +`agents.`. The validator checks that import against declarations +before the workflow image starts. + +## Agent loading and execution + +The local controller effectively performs: + +```python +module = load(entrypoint) +agent_class = getattr(module, configured_name) +agent = agent_class() +result = getattr(agent, method_name)(**args) +``` + +Consequences: + +- The class is module-level and named exactly as configured. +- Construction takes no arguments. +- Declared methods accept yaml argument names as keyword arguments. +- Methods are synchronous; this path does not await a coroutine. +- Dicts and lists are JSON-encoded before entering Redis; other results become + strings. +- A remote Future's `.value()` returns text, not the original Python object. + +Agent import and construction exceptions are caught by the controller. A failed +agent may still advertise healthy because health is written independently of +successful agent loading. That is why image probes import both the runtime and +the entrypoint explicitly. + +## Workflow execution + +The workflow file is executed, not imported. Therefore: + +- module-level code runs at container startup; +- `__name__ == "__main__"`; +- `deploy()` blocks in the web server; +- the workflow function runs once per request; +- its function name determines the REST route exposed by the compatibility + runtime. + +The deployment platform additionally expects `/main` with a `{query: string}` +body. This platform constraint is stricter than the underlying transport. + +Each stub method returns a Future immediately. `.value()` blocks. Dispatching +and resolving inside one comprehension serializes work without raising an +error; dispatch all calls first, then resolve them. + +The workflow container also starts runtime controller code and has its own +package resolution. Probe it independently from agent images. + +## Build context and collisions + +The runtime copies project files while preserving relative paths, then writes +shared runtime modules, generated stubs, and entrypoints into the image. Later +writes can shadow project files. + +Avoid root project modules named like runtime files, including: + +```text +future.py +ventis_context.py +local_controller.py +local_controller_frontend.py +redis_client.py +grpc_options.py +bedrock.py +deploy.py +session_logging.py +workflow_launcher.py +``` + +Also avoid a yaml basename that shadows a different source module imported by an +adapter. The validator checks deterministic flat-name collisions. + +File sweep and editable-install behavior are runtime capabilities. For nested +imports, follow [packaging.md](packaging.md). + +## Dependencies and protobuf + +Agent and workflow images include a small runtime dependency set. Config +`requirements` adds source-specific distributions. A malformed requirements +value can be normalized away while image generation continues; missing imports +then surface only when the agent loads. + +The build compiles gRPC Python stubs on the host and copies them into images. +The image resolver does not necessarily know the generated-code version. A +source dependency that constrains protobuf below the host generator version can +produce a green image build that dies on: + +```text +import local_controller +``` + +Always run that probe before probing the entrypoint. Treat a generated-code / +runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter +source dependencies silently. + +## Credentials capability + +When `env_file` capability is available, the top-level config path is resolved +against the project root and passed at container start. Hidden env files are not +copied into images. Invalid paths are deploy-preflight errors. + +When the capability is unavailable, declaring `env_file` has no effect. If the +source needs credentials, report the capability blocker rather than hardcoding +or vendoring a secret. + +A source that constructs its client at import time works only when credentials +are already in the container environment. Image entrypoint probes therefore use +the same env file as deployment. + +For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). + +## Policy and provider behavior + +No policy file means unrestricted service access. If a policy exists, it needs a +non-empty rules list. Rules are evaluated by specificity and first match; +services excluded from the selected rule fail after request acceptance. + +Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior +and remote networking are covered in [ec2.md](ec2.md). + +## Cleanup boundary + +Stopping foreground deploy normally invokes controller cleanup for recorded +containers and Redis. Hard kills and failures before resource registration may +leave resources behind. + +`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and +`docker_container/`. It does not remove containers or images. Remove exact +leftovers explicitly and preserve source, port scaffolding, and requested +evidence. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md new file mode 100644 index 0000000..361acf9 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md @@ -0,0 +1,60 @@ +# Troubleshooting + +Read this after a failed build, image probe, deploy, or request. For mechanisms, +read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, +read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). + +## Build or deploy stops early + +| Symptom | Likely cause | +|---|---| +| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | +| Two services produce one image | Config names collide after lowercase normalization | +| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | +| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | +| Replica conversion `TypeError` | `replicas` is not an integer | +| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | +| Port or container name already in use | A previous deployment did not complete cleanup | + +## Container exits or serves nothing + +| Symptom | Likely cause | +|---|---| +| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | +| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | +| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | +| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | +| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | +| Third-party module is missing | Distribution is absent from source metadata and config requirements | +| Stub import raises `NameError` | yaml argument type is not a bare builtin | +| Source module behaves like an empty stub | A generated stub basename shadowed the source module | +| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | + +## Request is accepted, then fails + +| Symptom | Likely cause | +|---|---| +| Unexpected keyword argument | yaml argument name differs from adapter parameter name | +| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | +| Unauthorized service | The first matching policy rule excludes that service | +| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | +| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | +| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | +| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | +| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | + +## Deployment platform endpoint + +| Symptom | Likely cause | +|---|---| +| 404 while workflow container is healthy | Workflow function is not named `main` | +| 400 before host receives request | Body is not the platform's `{query: string}` shape | +| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | + +## Cleanup + +| Symptom | Likely cause | +|---|---| +| `ventis clean` succeeds but containers remain | The command removes generated directories only | +| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | +| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py b/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py new file mode 100755 index 0000000..04baf37 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py @@ -0,0 +1,1257 @@ +#!/usr/bin/env python3 +"""Preflight the runtime traps that `ventis build` cannot see. + +This deliberately does not duplicate build-time validation such as malformed +YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those +checks. This script parses Python without importing it and catches failures that +otherwise stay hidden until a container loads an agent, starts a workflow, or +serves its first request. A replica is not evidence: the controller writes +`healthy` to Redis before `_load_agent` runs. + + python validate.py [project_dir] [-c config/global_controller.yaml] + [--json] [--strict] + +Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. + +Runtime capabilities vary across CanyonOS Core installations. This script probes +the importable `ventis` package directly. A capability-gated check reports +UNAVAILABLE when its behavior cannot be proven. +""" + +import argparse +import ast +import builtins +import json +import os +import re +import sys +from typing import ClassVar + +try: + import yaml +except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency + sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") + raise SystemExit(2) from None + + +DEFAULT_CONFIG_PATH = "config/global_controller.yaml" + +# Copied flat into every image over the swept project tree, so a project module +# landing flat under one of these names is overwritten. +# ventis/stub_generator.py generate_docker / generate_workflow_docker. +RUNTIME_FLAT_NAMES = frozenset( + { + "future.py", + "ventis_context.py", + "local_controller.py", + "local_controller_frontend.py", + "redis_client.py", + "grpc_options.py", + "gpu_metrics.py", + "bedrock.py", + "deploy.py", + "session_logging.py", + "workflow_launcher.py", + } +) + +# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. +BASE_AGENT_REQUIREMENTS = [ + "grpcio", + "grpcio-tools", + "redis", + "pyyaml", + "psutil", + "ipdb", + "ipython", + "boto3", +] +# Import name -> distribution name, for the handful where they differ and the +# mismatch would otherwise be reported as a missing requirement. +IMPORT_TO_DISTRIBUTION = { + "attr": "attrs", + "bs4": "beautifulsoup4", + "cv2": "opencv-python", + "dateutil": "python-dateutil", + "dotenv": "python-dotenv", + "grpc": "grpcio", + "grpc_tools": "grpcio-tools", + "jwt": "pyjwt", + "PIL": "pillow", + "psycopg": "psycopg", + "psycopg2": "psycopg2-binary", + "pydantic_settings": "pydantic-settings", + "sklearn": "scikit-learn", + "typing_extensions": "typing-extensions", + "yaml": "pyyaml", +} + +SECRET_PATTERNS = [ + (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), + (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), + (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), + (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), +] +SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) + +ERROR = "ERROR" +WARN = "WARN" +INFO = "INFO" + + +# ------------------------------------------------------------------ # +# Capabilities # +# ------------------------------------------------------------------ # +# +# Stable labels for behavior detected from the importable runtime. They contain +# no external development metadata. + +CAPABILITY_SOURCE = { + "env_file": "runtime env-file injection", + "editable_install": "editable project installation", + "sweeps_all_files": "full project-file sweep", + "stub_two_destinations": "flat and package stub destinations", +} + + +def probe_capabilities(): + """Ask the importable ventis package what it actually supports.""" + caps = dict.fromkeys(CAPABILITY_SOURCE, False) + caps["ventis"] = False + try: + from ventis import stub_generator + except Exception: # noqa: BLE001 - a broken install must not crash the check + return caps + + caps["ventis"] = True + caps["editable_install"] = hasattr(stub_generator, "_install_step") + caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") + caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") + + import importlib + + for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): + try: + module = importlib.import_module(module_name) + except Exception: # noqa: BLE001,S112 - the other path is the live one + continue + if hasattr(module, "resolve_env_file"): + caps["env_file"] = True + break + return caps + + +# ------------------------------------------------------------------ # +# YAML with line numbers # +# ------------------------------------------------------------------ # + + +class LineDict(dict): + """A mapping that remembers where it and each of its keys were written.""" + + line = 0 + key_lines: ClassVar[dict] = {} + + +class LineLoader(yaml.SafeLoader): + pass + + +def _construct_mapping(loader, node): + data = LineDict() + yield data + data.update(loader.construct_mapping(node, deep=False)) + data.line = node.start_mark.line + 1 + data.key_lines = { + key.value: key.start_mark.line + 1 + for key, _ in node.value + if isinstance(key, yaml.ScalarNode) + } + + +LineLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping +) + + +def line_of(mapping, key=None): + """The source line of `key` inside `mapping`, or of the mapping itself.""" + if not isinstance(mapping, LineDict): + return 0 + if key is not None: + return mapping.key_lines.get(key, mapping.line) + return mapping.line + + +def load_yaml(path): + """Parse `path`, returning (data, error). Never raises.""" + try: + with open(path, "r", encoding="utf-8") as handle: + return yaml.load(handle, Loader=LineLoader), None + except Exception as exc: # noqa: BLE001 - any parse failure is a finding + return None, str(exc) + + +# ------------------------------------------------------------------ # +# Findings # +# ------------------------------------------------------------------ # + + +class Report: + def __init__(self, project_dir, capabilities): + self.project_dir = project_dir + self.capabilities = capabilities + self.findings = [] + # A peer agent is imported by the name of its generated stub, which the + # build copies flat into every image. Those are not project modules and + # need no requirement. + self.stub_module_names = set() + + def add(self, check, level, path, line, summary, mechanism): + self.findings.append( + { + "check": check, + "level": level, + "path": self.rel(path) if path else "", + "line": line or 0, + "summary": summary, + "mechanism": mechanism, + } + ) + + def error(self, check, path, line, summary, mechanism): + self.add(check, ERROR, path, line, summary, mechanism) + + def warn(self, check, path, line, summary, mechanism): + self.add(check, WARN, path, line, summary, mechanism) + + def unavailable(self, check, summary): + self.add(check, INFO, "", 0, summary, "") + + def rel(self, path): + try: + return os.path.relpath(path, self.project_dir) + except ValueError: + return path + + def counts(self): + errors = sum(1 for f in self.findings if f["level"] == ERROR) + warnings = sum(1 for f in self.findings if f["level"] == WARN) + return errors, warnings + + +# ------------------------------------------------------------------ # +# Python source helpers # +# ------------------------------------------------------------------ # + + +def parse_python(path): + """AST for `path`, or (None, error). The port is never imported.""" + try: + with open(path, "r", encoding="utf-8") as handle: + source = handle.read() + except OSError as exc: + return None, str(exc) + try: + return ast.parse(source, filename=path), None + except SyntaxError as exc: + return None, f"{exc.msg} (line {exc.lineno})" + + +def find_class(tree, name): + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == name: + return node + return None + + +def class_methods(class_node): + return { + node.name: node + for node in class_node.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + + +def parameter_names(func_node): + """Every parameter a caller can pass by keyword, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + return positional + [a.arg for a in args.kwonlyargs] + + +def required_parameters(func_node): + """Parameters with no default, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + if args.defaults: + positional = positional[: len(positional) - len(args.defaults)] + kwonly = [ + arg.arg + for arg, default in zip(args.kwonlyargs, args.kw_defaults) + if default is None + ] + return positional + kwonly + + +def toplevel_import_names(tree): + """Top-level package name of every import in the module, with line numbers.""" + names = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + names.setdefault(alias.name.split(".")[0], node.lineno) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + names.setdefault(node.module.split(".")[0], node.lineno) + return names + + +# ------------------------------------------------------------------ # +# V006-V010 adapter failures hidden by _load_agent # +# ------------------------------------------------------------------ # + +BUILTIN_TYPE_NAMES = frozenset( + name + for name in dir(builtins) + if isinstance(getattr(builtins, name), type) and not name[0].isupper() +) + + +def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): + """V006 V007 V008 V009 V010.""" + name = agent_block["name"] + functions = agent_block.get("functions") or [] + check_argument_types(report, agent_yaml_path, functions) + + entrypoint = entry.get("entrypoint") + if not entrypoint: + return + entrypoint_path = os.path.join(project_dir, entrypoint) + if not os.path.isfile(entrypoint_path): + return + + tree, error = parse_python(entrypoint_path) + if tree is None: + report.error( + "V006", + entrypoint_path, + 0, + f"the entrypoint does not parse: {error}", + "_load_agent exec_module's it and swallows the exception; the first " + "request answers 'No agent loaded'.", + ) + return + + class_node = find_class(tree, name) + if class_node is None: + classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] + found = ", ".join(classes) if classes else "no classes at all" + report.error( + "V006", + entrypoint_path, + 1, + f"no class named `{name}` at module level (found: {found})", + "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " + "the AttributeError. The class name must equal agent.name exactly.", + ) + return + + methods = class_methods(class_node) + check_constructor(report, entrypoint_path, name, methods) + + for func in functions: + if not isinstance(func, dict) or not isinstance(func.get("name"), str): + continue + check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) + + +def check_argument_types(report, agent_yaml_path, functions): + """V010 -- the type string is pasted into an ast.Name, never checked.""" + for func in functions: + if not isinstance(func, dict): + continue + for arg in func.get("arguments") or []: + if not isinstance(arg, dict) or "type" not in arg: + continue + declared = arg.get("type") + if not isinstance(declared, str): + continue # stub generation reports malformed type values + if declared in BUILTIN_TYPE_NAMES: + continue + report.error( + "V010", + agent_yaml_path, + line_of(arg, "type"), + f"`type: {declared}` is not a builtin", + "stub_generator pastes it verbatim into the generated " + "annotation, and the stub module imports only Future and " + "inspect. Anything else raises NameError when the stub is " + "imported -- after a green build. Use str int float bool dict " + "list.", + ) + + +def check_constructor(report, entrypoint_path, name, methods): + """V007 -- _load_agent calls agent_class() with no arguments.""" + init = methods.get("__init__") + if init is None: + return + required = required_parameters(init) + if required: + report.error( + "V007", + entrypoint_path, + init.lineno, + f"`{name}.__init__` requires {', '.join(required)}", + "_load_agent calls agent_class() with no arguments; the TypeError is " + "swallowed and the first request answers 'No agent loaded'. Read " + "configuration from the environment inside __init__ instead.", + ) + + +def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): + """V008 V009.""" + func_name = func["name"] + method = methods.get(func_name) + if method is None: + report.error( + "V008", + entrypoint_path, + 0, + f"`{class_name}` has no method `{func_name}`", + "The yaml declares it, so callers get a stub for it; the controller " + f"then answers \"Agent {class_name} has no method '{func_name}'\".", + ) + return + + # V009 -- nothing on the execution path awaits. + if isinstance(method, ast.AsyncFunctionDef): + report.error( + "V009", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` is `async def`", + "The executor calls method(**args) with no await, so Redis receives " + "''. Keep the signature synchronous and call " + "asyncio.run(...) inside the body.", + ) + + # V008 -- the controller calls method(**args) with the yaml's names. + declared = [ + arg["name"] + for arg in func.get("arguments") or [] + if isinstance(arg, dict) and isinstance(arg.get("name"), str) + ] + actual = parameter_names(method) + required = required_parameters(method) + + missing = [arg for arg in declared if arg not in actual] + if missing: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` has no parameter " + f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", + "LocalController does method(**args) with the yaml's argument names. " + "A mismatch is TypeError: unexpected keyword argument, at request " + f"time. See {os.path.basename(agent_yaml_path)}.", + ) + + unfilled = [arg for arg in required if arg not in declared] + if unfilled: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " + "the yaml does not declare", + "Only declared arguments are ever sent, and the generated stub gives " + "none of them a default. Declare them in the yaml or default them " + "in the signature.", + ) + + + +# ------------------------------------------------------------------ # +# V016-V018 the workflow # +# ------------------------------------------------------------------ # + + +def check_stub_imports(report, workflow_path, tree, stub_classes): + """V023 -- the workflow must import a stub as `from agents. import `. + + The build copies each stub to exactly one path, and for the workflow image + that path is agents/.py. Two ways of writing this line fail, and + the project walks you into both: the flat form is what examples/ uses, and + the class name is the one `ventis build` prints, which is not the one it + writes. + """ + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + base = alias.name.split(".")[0] + if base in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`import {alias.name}` -- the stub is at " + f"agents/{base}.py, not flat", + "The build copies a stub to one path, and for the " + "workflow that path is under agents/. This is a " + "ModuleNotFoundError the moment the workflow runs. " + f"Write `from agents.{base} import {stub_classes[base]}`.", + ) + continue + + if not isinstance(node, ast.ImportFrom) or not node.module: + continue + + module = node.module + if module in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`from {module} import ...` -- the stub is at " + f"agents/{module}.py, not flat", + "The build copies a stub to one path, and for the workflow " + "that path is under agents/. The flat form is what this " + "repository's own examples use and it raises " + "ModuleNotFoundError in the workflow image. Write " + f"`from agents.{module} import {stub_classes[module]}`.", + ) + continue + + if not module.startswith("agents."): + continue + base = module.split(".", 1)[1] + expected = stub_classes.get(base) + if expected is None: + continue + for alias in node.names: + if alias.name == expected: + continue + if alias.name == f"{expected}Stub": + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is the name the build prints, not the " + f"class it writes", + "generate_agent_stub sets class_name = agent_config['name'] " + "and then recomputes it with a 'Stub' suffix for the log " + "line only. The message names a class that does not exist; " + f"the class is `{expected}`.", + ) + else: + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is not a class the stub for {base} defines", + f"The stub's class carries the agent's own name: `{expected}`.", + ) + + +def check_workflow(report, workflow_path, stub_classes=None): + """V016 V017 V018 V023.""" + tree, error = parse_python(workflow_path) + if tree is None: + report.error("V016", workflow_path, 0, f"does not parse: {error}", "") + return + + main = None + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( + node.name == "main" + ): + main = node + break + + if main is None: + defined = [ + n.name + for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + found = ", ".join(defined) if defined else "no top-level functions" + report.error( + "V016", + workflow_path, + 1, + f"no top-level function named `main` (found: {found})", + "CanyonOS Core serves POST /, but the deployment platform's " + "test endpoint posts to a hardcoded /main. A differently named " + "workflow builds, deploys and stays unreachable -- 404, container " + "healthy.", + ) + else: + check_main_signature(report, workflow_path, main) + + if stub_classes: + check_stub_imports(report, workflow_path, tree, stub_classes) + + # V016 -- deploy() is what starts Flask. + if not any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "deploy" + for node in ast.walk(tree) + ): + report.error( + "V016", + workflow_path, + 1, + "the workflow never calls `deploy(...)`", + "workflow_launcher.py exec's this file and nothing else starts the " + "HTTP server; the container comes up serving nothing.", + ) + + check_main_guard(report, workflow_path, tree) + check_fused_fanout(report, workflow_path, tree) + + +def check_main_signature(report, workflow_path, main): + """V016 -- the platform sends exactly {"query": ...}.""" + if isinstance(main, ast.AsyncFunctionDef): + report.error( + "V016", + workflow_path, + main.lineno, + "`main` is `async def`", + "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " + "no await; the response body would be a coroutine repr.", + ) + params = parameter_names(main) + if not params: + report.error( + "V016", + workflow_path, + main.lineno, + "`main` takes no arguments", + 'The platform posts {"query": "..."} and deploy() splats the ' + "body in as kwargs -- TypeError on every request.", + ) + return + if params[0] != "query": + report.error( + "V016", + workflow_path, + main.lineno, + f"`main`'s first parameter is `{params[0]}`, not `query`", + "The platform's body schema is strictly validated as " + "{query: string}; any other key is rejected with 400 in the control " + "plane, before the request reaches the host.", + ) + extra = [p for p in required_parameters(main) if p != "query"] + if extra: + report.error( + "V016", + workflow_path, + main.lineno, + f"`main` requires {', '.join(extra)} beyond `query`", + "Only `query` is ever sent, so every other parameter needs a " + "default or the call raises on every request. Pack richer input " + "into `query`.", + ) + + +def check_main_guard(report, workflow_path, tree): + """V017 -- the workflow is exec'd, so __name__ == "__main__".""" + for node in tree.body: + if not isinstance(node, ast.If): + continue + test = node.test + if ( + isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and test.left.id == "__name__" + and any( + isinstance(c, ast.Constant) and c.value == "__main__" + for c in test.comparators + ) + ): + report.error( + "V017", + workflow_path, + node.lineno, + '`if __name__ == "__main__":` block in the workflow', + "workflow_launcher.py runs exec(open().read()), so " + "__name__ IS '__main__' here and this block executes in " + "production, at container start.", + ) + + +def check_fused_fanout(report, workflow_path, tree): + """V018 -- .value() blocks, so dispatching and resolving in one + comprehension runs the fan-out one call at a time.""" + comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) + for node in ast.walk(tree): + if not isinstance(node, comprehensions + (ast.DictComp,)): + continue + elements = ( + [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] + ) + for element in elements: + for inner in ast.walk(element): + if ( + isinstance(inner, ast.Call) + and isinstance(inner.func, ast.Attribute) + and inner.func.attr == "value" + and isinstance(inner.func.value, ast.Call) + ): + report.error( + "V018", + workflow_path, + node.lineno, + "one comprehension both dispatches a call and resolves " + "it with .value()", + ".value() blocks, so each call completes before the " + "next is dispatched. It does not error -- the fan-out " + "is just silently serial, and with it the reason to be " + "on CanyonOS Core. Dispatch every call first, then resolve: " + "futures = [a.work(i) for i in items] then " + "[f.value() for f in futures].", + ) + return + + +# ------------------------------------------------------------------ # +# V019-V020 what the copy order overwrites # +# ------------------------------------------------------------------ # + + +def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): + """V019 V020 -- later copies land on earlier ones at the context root.""" + for entry in sorted(os.listdir(project_dir)): + path = os.path.join(project_dir, entry) + if not os.path.isfile(path) or not entry.endswith(".py"): + continue + + # V019 -- the shared runtime is copied flat, after the project sweep. + if entry in RUNTIME_FLAT_NAMES: + report.error( + "V019", + path, + 1, + f"a project module named `{entry}` sits at the project root", + "The shared CanyonOS Core runtime is copied flat into the image after " + "the project sweep, so this file is overwritten by CanyonOS Core's own " + f"{entry}. Rename it or move it into a package directory.", + ) + + # V020 -- a stub is copied flat under its yaml's basename. + entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} + for yaml_path in yaml_paths: + stem = os.path.splitext(os.path.basename(yaml_path))[0] + module = f"{stem}.py" + if module in entrypoint_basenames: + continue # the entrypoint is copied last and wins its flat name back + candidate = os.path.join(project_dir, module) + if os.path.isfile(candidate): + report.error( + "V020", + candidate, + 1, + f"`{os.path.basename(yaml_path)}` generates a stub that lands on " + f"`{module}`", + "The yaml's basename names the stub, and the stub is copied flat " + "over the swept tree. Anything importing this module inside the " + "container gets the generated stub instead of the real code. " + "Rename the yaml to match its own entrypoint.", + ) + + +# ------------------------------------------------------------------ # +# V030-V031 capability-gated rules # +# ------------------------------------------------------------------ # + + +def check_env_file(report, config, config_path, project_dir): + """V030 -- gated on detected env-file injection support.""" + declared = config.get("env_file") + supported = report.capabilities.get("env_file") + + if not supported: + if declared: + report.error( + "V030", + config_path, + line_of(config, "env_file"), + f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", + "No resolve_env_file in the importable ventis package, so the " + "key is silently dropped and the container answers a provider " + "credential error on the first request. This port requires the " + "`env_file` runtime capability.", + ) + else: + report.unavailable( + "V030", + "env_file is not supported by the importable `ventis` runtime. " + "Credentials have no declared path into a container on this tree.", + ) + return + + if not declared: + report.warn( + "V030", + config_path, + line_of(config), + "no `env_file:` in the config", + "Only runtime-managed VENTIS_* variables are guaranteed without it. " + "If the source reads credentials from the environment, the first " + "request fails on a provider error.", + ) + return + + # Path existence and readability are deploy-preflight checks. Do not + # duplicate them here. + + +def check_import_root(report, project_dir, entrypoint_paths): + """V031 -- gated on detected editable-install support.""" + supported = report.capabilities.get("editable_install") + has_metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + + non_flat = [] + for path in entrypoint_paths: + tree, _ = parse_python(path) + if tree is None: + continue + for name, lineno in toplevel_import_names(tree).items(): + if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: + continue + if _resolves_flat(project_dir, name): + continue + location = _resolves_nested(project_dir, name) + if location: + non_flat.append((path, lineno, name, location)) + + if not supported: + report.unavailable( + "V031", + "the editable install (`-e .`) is not supported by the importable " + "`ventis` runtime. Only names rooted at /app import inside a container.", + ) + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, which is not at the " + "project root", + "sys.path[0] is /app and this CanyonOS Core runs no editable install, " + "so only modules swept to the root import. The adapter raises " + "ModuleNotFoundError inside _load_agent and the first request " + "answers 'No agent loaded'.", + ) + return + + if non_flat and not has_metadata: + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, and the project root " + "has no packaging metadata", + "A pyproject.toml, setup.py or setup.cfg at the port root is " + "what adds `-e .`; metadata nested inside the untouched source " + "tree is ignored. Add minimal root metadata pointing at the " + "existing package directory. Without it the install is skipped " + "silently.", + ) + + +def _resolves_flat(project_dir, name): + """Whether Python can resolve `name` with /app as its import root. + + A directory does not need __init__.py: PEP 420 namespace packages resolve + from sys.path just like regular packages. + """ + return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( + os.path.join(project_dir, name) + ) + + +def _resolves_nested(project_dir, name): + """Where below /app `name` lives but cannot resolve as a top-level name.""" + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] + if root == project_dir: + continue + if f"{name}.py" in files: + return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) + if name in dirs: + return os.path.relpath(os.path.join(root, name), project_dir) + return None + + +# ------------------------------------------------------------------ # +# W003, W006 secrets and imports a green build does not reject # +# ------------------------------------------------------------------ # + + +def check_secrets(report, port_paths): + """W003 -- env_file is the way in; nothing else is.""" + for path in port_paths: + try: + with open(path, "r", encoding="utf-8") as handle: + lines = handle.read().splitlines() + except OSError: + continue + for number, line in enumerate(lines, start=1): + for pattern, description in SECRET_PATTERNS: + if pattern.search(line): + report.warn( + "W003", + path, + number, + f"this line looks like {description}", + "Never put a secret in the source tree or the build " + "context. The build sweeps the project into every " + "image.", + ) + break + + tree, _ = parse_python(path) + if tree is None: + continue + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + if not ( + isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + and node.value.value.strip() + ): + continue + for target in node.targets: + if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): + report.warn( + "W003", + path, + node.lineno, + f"`{target.id}` is assigned a literal string", + "Read it from the environment instead; the build sweeps " + "this file into every image.", + ) + + +def _pyproject_dependencies(project_dir): + """What `-e .` installs alongside `requirements:`, or None if unreadable. + + None and the empty set mean different things here: empty means the project + declares no dependencies, None means we could not find out -- a setup.py, or + a tomllib this interpreter does not have. The caller must not treat the + second as the first, or it warns about imports the install would satisfy. + """ + path = os.path.join(project_dir, "pyproject.toml") + if not os.path.isfile(path): + return None + try: + import tomllib + except ImportError: # < 3.11 + return None + try: + with open(path, "rb") as handle: + data = tomllib.load(handle) + except Exception: # noqa: BLE001 - malformed metadata is uv's error to give + return None + deps = (data.get("project") or {}).get("dependencies") + if not isinstance(deps, list): + return None + return {_normalize_distribution(d) for d in deps if isinstance(d, str)} + + +def check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path +): + """W006 -- an import the container cannot satisfy.""" + tree, _ = parse_python(entrypoint_path) + if tree is None: + return + + declared = { + _normalize_distribution(item) + for item in (entry.get("requirements") or []) + if isinstance(item, str) + } + + # Where the editable install exists, `-e .` resolves the project's own + # [project.dependencies] in the same pass as `requirements:`. Warning about + # those is a false positive, and a false warning about a dependency is worse + # than none: it teaches the reader to dismiss this check. + editable = report.capabilities.get("editable_install") + metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + unreadable_metadata = False + if editable and metadata: + project_deps = _pyproject_dependencies(project_dir) + if project_deps is None: + unreadable_metadata = True + else: + declared |= project_deps + base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} + stdlib = getattr(sys, "stdlib_module_names", frozenset()) + + for name, lineno in sorted(toplevel_import_names(tree).items()): + if name in stdlib or name == "ventis": + continue + # Provided by the image itself: the shared runtime is copied flat, and + # every agents/*.yaml generates a stub that is copied flat too. + if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: + continue + if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): + continue + distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) + if distribution in base or distribution in declared: + continue + if unreadable_metadata: + mechanism = ( + "The container installs the base list, `requirements:`, and -- " + "since this project declares packaging metadata -- whatever " + "`-e .` resolves from it. That metadata could not be read here, " + f"so if it already requires `{name}` this line is noise; " + "otherwise it is a ModuleNotFoundError inside _load_agent and " + "'No agent loaded' on the first request." + ) + else: + mechanism = ( + "The container installs the base list plus `requirements:` and " + "nothing else, so this is a ModuleNotFoundError inside " + "_load_agent and 'No agent loaded' on the first request. If the " + f"distribution is named something other than `{name}`, declare " + f"that name in {report.rel(config_path)}." + ) + report.warn( + "W006", + entrypoint_path, + lineno, + f"`import {name}` is in neither the runtime's base list nor this " + "entry's `requirements:`", + mechanism, + ) + + +def _normalize_distribution(name): + return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( + "_", "-" + ) + + +# ------------------------------------------------------------------ # +# Driver # +# ------------------------------------------------------------------ # + + +def validate(project_dir, config_path, capabilities): + """Inspect only failures hidden behind a successful image build.""" + report = Report(project_dir, capabilities) + + # The build owns config/YAML syntax and shape validation. We read only enough + # valid structure to locate code for the deeper checks below. + config, error = load_yaml(config_path) + if error is not None or not isinstance(config, dict): + report.unavailable( + "BUILD", + "runtime preflight skipped because the config cannot be read; " + "ventis build owns and reports this error.", + ) + return report + + import glob + + yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) + report.stub_module_names = { + os.path.splitext(os.path.basename(path))[0] for path in yaml_paths + } + + agents_by_name = {} + stub_classes = {} + for path in yaml_paths: + data, yaml_error = load_yaml(path) + agent = data.get("agent") if isinstance(data, dict) else None + name = agent.get("name") if isinstance(agent, dict) else None + if yaml_error is not None or not isinstance(name, str): + continue # ventis build reports malformed agent declarations + agents_by_name[name] = (path, agent) + stub_classes[os.path.splitext(os.path.basename(path))[0]] = name + + entries = config.get("agents") + if not isinstance(entries, list): + report.unavailable( + "BUILD", + "runtime preflight skipped because `agents:` is not a list; " + "ventis build owns and reports this error.", + ) + return report + + entrypoints = [] + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": + continue + name = entry.get("name") + entrypoint = entry.get("entrypoint") + if isinstance(entrypoint, str): + entrypoints.append(entrypoint) + if name in agents_by_name: + yaml_path, agent_block = agents_by_name[name] + check_adapter(report, yaml_path, agent_block, entry, project_dir) + entrypoint_path = os.path.join(project_dir, entrypoint or "") + if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): + check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path + ) + + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": + continue + workflow_file = entry.get("workflow_file") + if not isinstance(workflow_file, str): + continue + workflow_path = os.path.join(project_dir, workflow_file) + if os.path.isfile(workflow_path): + check_workflow(report, workflow_path, stub_classes) + + # These survive a green build and otherwise surface only in a container or + # on its first request. + check_flat_collisions(report, project_dir, yaml_paths, entrypoints) + check_env_file(report, config, config_path, project_dir) + + entrypoint_paths = [ + os.path.join(project_dir, e) + for e in entrypoints + if os.path.isfile(os.path.join(project_dir, e)) + ] + check_import_root(report, project_dir, entrypoint_paths) + + port_paths = list(entrypoint_paths) + for entry in entries: + if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): + candidate = os.path.join(project_dir, entry["workflow_file"]) + if os.path.isfile(candidate): + port_paths.append(candidate) + + # Secret detection remains because a green image build would permanently + # bake the credential into every image. + check_secrets(report, port_paths) + return report + + + +# ------------------------------------------------------------------ # +# Output # +# ------------------------------------------------------------------ # + +LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} + + +def _wrap(text, width, indent): + words = text.split() + lines = [] + current = "" + for word in words: + candidate = f"{current} {word}".strip() + if len(candidate) + len(indent) > width and current: + lines.append(indent + current) + current = word + else: + current = candidate + if current: + lines.append(indent + current) + return lines + + +def print_report(report, project_dir): + caps = report.capabilities + if not caps.get("ventis"): + print("ventis is not importable here -- capability-gated rules are") + print("reported UNAVAILABLE rather than checked.\n") + else: + print("CanyonOS Core capabilities detected:") + for key, source in CAPABILITY_SOURCE.items(): + mark = "yes" if caps.get(key) else "no " + print(f" {mark} {key:<22} {source}") + print() + + findings = sorted( + report.findings, + key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), + ) + for finding in findings: + where = finding["path"] + if where and finding["line"]: + where = f"{where}:{finding['line']}" + header = f"{finding['check']} {finding['level']:<5}" + print(f"{header} {where}" if where else header) + for line in _wrap(finding["summary"], 78, " "): + print(line) + if finding["mechanism"]: + for line in _wrap(finding["mechanism"], 78, " "): + print(line) + print() + + errors, warnings = report.counts() + if not findings: + print(f"{project_dir}: clean.") + return + print(f"{errors} error(s), {warnings} warning(s).") + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Check a CanyonOS Core port against the rules in SKILL.md." + ) + parser.add_argument( + "project_dir", nargs="?", default=".", help="the port's project root" + ) + parser.add_argument( + "-c", + "--config", + default=DEFAULT_CONFIG_PATH, + help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", + ) + parser.add_argument("--json", action="store_true", help="emit findings as JSON") + parser.add_argument( + "--strict", action="store_true", help="fail on warnings as well as errors" + ) + args = parser.parse_args(argv) + + project_dir = os.path.abspath(args.project_dir) + config_path = ( + args.config + if os.path.isabs(args.config) + else os.path.join(project_dir, args.config) + ) + + capabilities = probe_capabilities() + report = validate(project_dir, config_path, capabilities) + errors, warnings = report.counts() + + if args.json: + print( + json.dumps( + { + "project_dir": project_dir, + "capabilities": capabilities, + "errors": errors, + "warnings": warnings, + "findings": report.findings, + }, + indent=2, + ) + ) + else: + print_report(report, report.rel(project_dir) or project_dir) + + if errors or (args.strict and warnings): + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/portfolio/agents/advisor_agent.py b/examples/portfolio/agents/advisor_agent.py index 0915eea..82d8936 100644 --- a/examples/portfolio/agents/advisor_agent.py +++ b/examples/portfolio/agents/advisor_agent.py @@ -2,7 +2,7 @@ # # Final stage. Turns the computed portfolio metrics and risk figures into a # short, plain-English briefing using a small, cheap model on AWS Bedrock -# (Converse API), called via ventis.llm.bedrock so token/cost telemetry gets +# (Converse API), called via ventis.controller.bedrock so token/cost telemetry gets # recorded onto this execution's future: hash. Configure # with env vars: # BEDROCK_MODEL_ID (default: meta.llama3-8b-instruct-v1:0) @@ -16,7 +16,7 @@ import os try: - from ventis.llm.bedrock import call_bedrock + from ventis.controller.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock diff --git a/examples/portfolio/agents/intent_agent.py b/examples/portfolio/agents/intent_agent.py index d74b27b..04124cf 100644 --- a/examples/portfolio/agents/intent_agent.py +++ b/examples/portfolio/agents/intent_agent.py @@ -7,7 +7,7 @@ # -> {"holdings": {"AAPL": 0.4, "MSFT": 0.35, "NVDA": 0.25}, # "lookback_days": 180} # -# Calls AWS Bedrock (Converse API) via ventis.llm.bedrock -- same pattern as +# Calls AWS Bedrock (Converse API) via ventis.controller.bedrock -- same pattern as # AdvisorAgent -- so token/cost telemetry gets recorded onto this execution's # future: hash. Configure with env vars: # BEDROCK_MODEL_ID (default: meta.llama3-8b-instruct-v1:0) @@ -25,7 +25,7 @@ import json try: - from ventis.llm.bedrock import call_bedrock + from ventis.controller.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index 96f371c..dbff7bb 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -16,7 +16,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/intent_agent.py - provider: EC2 + provider: local instance_type: t3.micro # Stage 0b: price history fetch. Network/IO-bound, cheap CPU. Called by @@ -28,7 +28,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/price_agent.py - provider: EC2 + provider: local instance_type: t3.micro requirements: [yfinance] @@ -41,7 +41,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/metrics_agent.py - provider: EC2 + provider: local instance_type: t3.micro # Stage 2: portfolio-level risk aggregation. Single call per request; needs @@ -53,7 +53,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/risk_agent.py - provider: EC2 + provider: local instance_type: t3.micro # Stage 3: LLM briefing via Bedrock. On the critical path, one call per @@ -65,7 +65,7 @@ agents: cpu: 1 memory: 512 entrypoint: agents/advisor_agent.py - provider: EC2 + provider: local instance_type: t3.micro # The workflow, exposed as a REST API. @@ -75,27 +75,23 @@ agents: redis_port: 6379 replicas: 1 workflow_file: workflow/portfolio_workflow.py - provider: EC2 + provider: local instance_type: t3.micro otel: + # The dashboard api's own OTLP ingest. Must be the full url including the + # path: the http exporter uses an explicitly-passed endpoint verbatim and + # only appends /v1/traces when reading OTEL_EXPORTER_OTLP_ENDPOINT. destinations: - - name: railway - protocol: grpc - endpoint: ${RAILWAY_OTLP_ENDPOINT} - insecure: true - headers: {} - - name: grafana + - name: local protocol: http - endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces - headers: - Authorization: Basic ${GRAFANA_OTLP_HEADERS} + endpoint: http://host.docker.internal:3000/v1/traces + headers: {} # Polling interval in seconds -poll_interval: 5 +poll_interval: 4 -# Redis connection redis: host: localhost port: 6379 @@ -111,5 +107,3 @@ ec2: ssh_user: ${EC2_SSH_USER} ssh_private_key_path: ${EC2_SSH_PRIVATE_KEY_PATH} -database: - url: ${DATABASE_URL} diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 619c4bd..8b0a76a 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -46,7 +46,7 @@ def main( advisor = AdvisorAgent() # Stage 0: parse the free-text request into structured holdings + window. - intent = intent_agent.parse(query=query) + intent = json.loads(intent_agent.parse(query=query).value()) holdings = intent["holdings"] lookback_days = intent["lookback_days"] diff --git a/examples/text2sql/agents/vllm_agent.py b/examples/text2sql/agents/vllm_agent.py index 4a7a245..ea23616 100644 --- a/examples/text2sql/agents/vllm_agent.py +++ b/examples/text2sql/agents/vllm_agent.py @@ -1,7 +1,7 @@ # VLLM Agent # # LLM backend for SQL candidate generation, called remotely by -# SQLGeneratorAgent. Calls AWS Bedrock (Converse API) via ventis.llm.bedrock +# SQLGeneratorAgent. Calls AWS Bedrock (Converse API) via ventis.controller.bedrock # so token/cost telemetry gets recorded onto this execution's # future: hash — same pattern as # examples/portfolio/agents/advisor_agent.py. @@ -14,7 +14,7 @@ import os try: - from ventis.llm.bedrock import call_bedrock + from ventis.controller.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock From 7cbd976185527e46019532766cc53deca51c7e79 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 12:24:43 -0700 Subject: [PATCH 26/31] removed some useless code --- SESSION_NOTES.md | 79 -- examples/epigenomics/README.md | 70 - examples/epigenomics/agents/dedup_agent.py | 44 - examples/epigenomics/agents/dedup_agent.yaml | 10 - examples/epigenomics/agents/filter_agent.py | 32 - examples/epigenomics/agents/filter_agent.yaml | 12 - examples/epigenomics/agents/index_agent.py | 30 - examples/epigenomics/agents/index_agent.yaml | 12 - examples/epigenomics/agents/map_agent.py | 33 - examples/epigenomics/agents/map_agent.yaml | 14 - examples/epigenomics/agents/sort_agent.py | 32 - examples/epigenomics/agents/sort_agent.yaml | 14 - examples/epigenomics/agents/split_agent.py | 27 - examples/epigenomics/agents/split_agent.yaml | 12 - .../epigenomics/config/global_controller.yaml | 76 - examples/epigenomics/config/policy.yaml | 20 - .../workflow/epigenomics_workflow.py | 97 -- examples/joke_writer/.car/app/.env.example | 20 - examples/joke_writer/.car/app/LICENSE | 21 - examples/joke_writer/.car/app/README.md | 177 --- .../joke_writer/.car/app/joke_workflow.py | 39 - examples/joke_writer/.car/app/joke_writer.py | 164 --- .../.car/config/global_controller.yaml | 52 - .../joke_writer/.car/config/joke_agent.yaml | 35 - examples/joke_writer/.env.example | 20 - examples/joke_writer/LICENSE | 21 - examples/joke_writer/README.md | 177 --- examples/joke_writer/joke_writer.py | 151 -- .../skills/porting-to-canyonos-core/SKILL.md | 240 ---- .../references/ec2.md | 41 - .../references/llm-proxy.md | 64 - .../references/packaging.md | 81 -- .../references/runtime-contract.md | 187 --- .../references/troubleshooting.md | 60 - .../porting-to-canyonos-core/validate.py | 1257 ----------------- 35 files changed, 3421 deletions(-) delete mode 100644 SESSION_NOTES.md delete mode 100644 examples/epigenomics/README.md delete mode 100644 examples/epigenomics/agents/dedup_agent.py delete mode 100644 examples/epigenomics/agents/dedup_agent.yaml delete mode 100644 examples/epigenomics/agents/filter_agent.py delete mode 100644 examples/epigenomics/agents/filter_agent.yaml delete mode 100644 examples/epigenomics/agents/index_agent.py delete mode 100644 examples/epigenomics/agents/index_agent.yaml delete mode 100644 examples/epigenomics/agents/map_agent.py delete mode 100644 examples/epigenomics/agents/map_agent.yaml delete mode 100644 examples/epigenomics/agents/sort_agent.py delete mode 100644 examples/epigenomics/agents/sort_agent.yaml delete mode 100644 examples/epigenomics/agents/split_agent.py delete mode 100644 examples/epigenomics/agents/split_agent.yaml delete mode 100644 examples/epigenomics/config/global_controller.yaml delete mode 100644 examples/epigenomics/config/policy.yaml delete mode 100644 examples/epigenomics/workflow/epigenomics_workflow.py delete mode 100644 examples/joke_writer/.car/app/.env.example delete mode 100644 examples/joke_writer/.car/app/LICENSE delete mode 100644 examples/joke_writer/.car/app/README.md delete mode 100644 examples/joke_writer/.car/app/joke_workflow.py delete mode 100644 examples/joke_writer/.car/app/joke_writer.py delete mode 100644 examples/joke_writer/.car/config/global_controller.yaml delete mode 100644 examples/joke_writer/.car/config/joke_agent.yaml delete mode 100644 examples/joke_writer/.env.example delete mode 100644 examples/joke_writer/LICENSE delete mode 100644 examples/joke_writer/README.md delete mode 100644 examples/joke_writer/joke_writer.py delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md delete mode 100755 examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py diff --git a/SESSION_NOTES.md b/SESSION_NOTES.md deleted file mode 100644 index 360efcc..0000000 --- a/SESSION_NOTES.md +++ /dev/null @@ -1,79 +0,0 @@ -# Session notes: canyonos serve, OTel pipeline, examples - -## Fixed (code changes, rebuilt+pushed `saakeths/canyonos:latest` where needed) - -1. **`ventis/stub_generator.py`**: stubs only ever got placed at ONE path - (nested-at-entrypoint, never flat), breaking any workflow that imports a - sibling agent directly (`from split_agent import SplitAgent`, e.g. - `examples/epigenomics`). Now placed at both. -2. **`otel_exporter.py`**: hardcoded Redis to `localhost`, but it runs inside - the GC container (bridge networking) while Redis is a sibling container — - crash-looped forever. Fixed to `host.docker.internal`. -3. **`ventis/OTLP_Exporter/db.py`**: `write_waiting_rows()` unconditionally - priced every future via a `aws_instance_pricing` table that only exists for - EC2 deployments — silently dropped **every** span for **every** - `provider: local` deployment, always. Wrapped cost lookups in try/except, - falls back to $0. -4. **`dashboard_stack.py`**: `web` port was hardcoded 8080 with no fallback. - Now searches for a free port (reuses an already-running dashboard's port - if one exists), same pattern as `init.py`'s GC port selection. -5. **`dashboard_stack.py`**: `database.url` was required; made optional - (dashboard boots fine with no DB configured in the project's own config). -6. **`dashboard.compose.yml`**: added `pull_policy: never` for the local-only - `canyonos-otel-receiver:local` image (compose was trying to pull it from a - registry that doesn't have it). Sequenced `otel-receiver` to start after - `api` (both raced to create `otel_spans`; `api`'s bare `CREATE TABLE` lost - the race and crashed). - -## Bundled (new, working) - -- `db` (plain Postgres) + `otel-receiver` (new `Dockerfile` for - `otlp_pg_receiver`, built locally as `canyonos-otel-receiver:local`) added - to `dashboard.compose.yml`. Verified real spans, sent via the actual OTLP - gRPC exporter, land in `otel_spans`. - -## Workarounds applied, NOT real fixes (will resurface) - -- **`canyonos quit` only tears down the GC container + volume**, never the - deployed agent/workflow/redis containers. Had to `docker rm -f` those by - exact name every time before a truly clean restart. -- **The named workspace volume is additive-only** (`docker cp`, never - clears) — files from a previous project leak into the next one's build - until you manually nuke the volume. -- **`otlp_pg_receiver` holds one Postgres connection with no reconnect - logic** — a DB restart silently kills every future write until the - receiver container itself is restarted. -- **`joke_writer/.car` layout** (`app/`, `config/`) doesn't match what - `ventis/cli.py`'s build step expects (`agents/`, flat entrypoints) — worked - around by manually copying files into the shape it wants. The real fix - (`nickhuo/car-artifact-layout`, already pushed) was not merged in. -- **joke_writer's LLM calls are stubbed to return `"animal"`** — for testing - only, real Bedrock creds needed to restore actual behavior (commented-out - code left in place). - -## Known, not touched - -- Pre-existing OrbStack local-provider startup race (first request right - after a container reports healthy can fail); a fix exists on an unrelated, - unmerged branch. -- Stale global `uv tool install` is a recurring trap — always - `uv tool install --reinstall .` after any `cli/` change. - -## What's still needed to actually see data in the UI - -The whole pipeline up to Postgres now genuinely works. **Nothing shows up in -the dashboard because `canyonos-api`/`canyonos-web` have zero code that reads -or displays `otel_spans`** — confirmed by inspecting their actual source -(they're a `cc-forge` rebrand: deploy/project management + a static -code-structure diagram, unrelated data model). To close the loop: - -1. New API route(s) in `canyon-code-forge/packages/api` that query - `otel_spans`. -2. New UI screen(s) in `canyon-code-forge/apps/web` to render it. -3. `web` currently has **no path to reach `api` at all** even once that - exists — no reverse proxy in its Caddyfile, and `api`'s port isn't - published to the host in `dashboard.compose.yml`. Needs one or the other - before the browser can fetch anything. - -All of the above is real feature work in a different repo, not a config or -wiring fix. diff --git a/examples/epigenomics/README.md b/examples/epigenomics/README.md deleted file mode 100644 index 19fb748..0000000 --- a/examples/epigenomics/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Epigenomics Example - -A synthetic, LLM-free workflow modeled on the -[WfCommons Epigenomics recipe](https://docs.wfcommons.org/en/latest/generating_workflows.html): -a split → fan-out (filter → align → sort) → fan-in (dedup) → index pipeline. -Every stage does deterministic SHA-256 work sized off chunk byte counts, so -results are reproducible and the fan-out width scales with `num_chunks` -- -useful for exercising scheduling/replica behavior locally without any real -model calls. - -## Pipeline - -``` -SplitAgent --> FilterAgent --> MapAgent --> SortAgent --\ - (1 call) (fan-out) (fan-out) (fan-out) --> DedupAgent --> IndexAgent - (fan-in barrier) (1 call) -``` - -- **SplitAgent** — splits `input_size` bytes into `num_chunks` equal chunks. -- **FilterAgent** — per-chunk contaminant filter (light cost). -- **MapAgent** — per-chunk alignment, the heaviest stage. -- **SortAgent** — per-chunk sort (moderate cost). -- **DedupAgent** — merges every sorted chunk into one digest (the barrier). -- **IndexAgent** — builds the final index from the merged digest. - -## Quick Start - -```bash -# Build stubs and Docker images -ventis build - -# Launch all agents -ventis deploy - -# Test with curl -curl -X POST http://:8080/main \ - -H 'Content-Type: application/json' \ - -d '{"input_size": 65536, "num_chunks": 4}' - -# Check result -curl http://:8080/status/ -``` - -## Project Structure - -``` -├── agents/ # Agent implementations and YAML definitions -│ ├── split_agent.py/.yaml -│ ├── filter_agent.py/.yaml -│ ├── map_agent.py/.yaml -│ ├── sort_agent.py/.yaml -│ ├── dedup_agent.py/.yaml -│ └── index_agent.py/.yaml -├── workflow/ # Workflow script (deployed as a REST API) -│ └── epigenomics_workflow.py -└── config/ - ├── global_controller.yaml # Deployment configuration (provider: local) - └── policy.yaml # Access control rules -``` - -## Policy Rules - -Edit `config/policy.yaml` to control which callers can access which agents. -Pass `_context` in your curl request to set the caller identity: - -```bash -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' \ - -d '{"input_size": 65536, "num_chunks": 4, "_context": {"origin": "admin"}}' -``` diff --git a/examples/epigenomics/agents/dedup_agent.py b/examples/epigenomics/agents/dedup_agent.py deleted file mode 100644 index d883854..0000000 --- a/examples/epigenomics/agents/dedup_agent.py +++ /dev/null @@ -1,44 +0,0 @@ -# Dedup Agent -# -# Fan-in barrier (mirrors Epigenomics' mark-duplicates/merge stage): needs -# every sorted chunk before it can run. Combines all chunk digests into one -# merged digest, with cost scaling off the total merged data volume. -# -# Resource profile: moderate CPU, single call per request (the barrier). - -import hashlib - - -class DedupAgent(object): - def __init__(self): - self.tools = [self.merge_dedup] - - def merge_dedup(self, chunks: list) -> dict: - """Merge and deduplicate every sorted chunk into one combined digest.""" - total_size = sum(c["size"] for c in chunks) - seed = "".join(c["digest"] for c in sorted(chunks, key=lambda c: c["chunk_id"])) - merged_digest = self._cpu_work(seed, total_size) - return { - "merged_digest": merged_digest, - "total_size": total_size, - "n_chunks": len(chunks), - } - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = DedupAgent() - print( - agent.merge_dedup( - [ - {"chunk_id": "chunk-0", "size": 16384, "digest": "aa"}, - {"chunk_id": "chunk-1", "size": 16384, "digest": "bb"}, - ] - ) - ) diff --git a/examples/epigenomics/agents/dedup_agent.yaml b/examples/epigenomics/agents/dedup_agent.yaml deleted file mode 100644 index 1c577cd..0000000 --- a/examples/epigenomics/agents/dedup_agent.yaml +++ /dev/null @@ -1,10 +0,0 @@ -agent: - name: DedupAgent - functions: - - name: merge_dedup - description: Merge and deduplicate every sorted chunk into one combined digest. - arguments: - - name: chunks - type: list - returns: - type: dict diff --git a/examples/epigenomics/agents/filter_agent.py b/examples/epigenomics/agents/filter_agent.py deleted file mode 100644 index 24dfc48..0000000 --- a/examples/epigenomics/agents/filter_agent.py +++ /dev/null @@ -1,32 +0,0 @@ -# Filter Agent -# -# First per-chunk stage in the fan-out (mirrors Epigenomics' filter_contams -# stage): scrubs one chunk and hands back a content digest the later stages -# build on. The "work" is a deterministic SHA-256 chain sized off the -# chunk's declared byte size, standing in for the real stage's per-byte cost. -# -# Resource profile: light CPU, high fan-out (one call per chunk). - -import hashlib - - -class FilterAgent(object): - def __init__(self): - self.tools = [self.filter_contams] - - def filter_contams(self, chunk_id: str, size: int) -> dict: - """Filter contaminants out of one chunk, returning its content digest.""" - digest = self._cpu_work(chunk_id, size) - return {"chunk_id": chunk_id, "size": size, "digest": digest} - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = FilterAgent() - print(agent.filter_contams("chunk-0", 16384)) diff --git a/examples/epigenomics/agents/filter_agent.yaml b/examples/epigenomics/agents/filter_agent.yaml deleted file mode 100644 index 9f10d29..0000000 --- a/examples/epigenomics/agents/filter_agent.yaml +++ /dev/null @@ -1,12 +0,0 @@ -agent: - name: FilterAgent - functions: - - name: filter_contams - description: Filter contaminants out of one chunk, returning its content digest. - arguments: - - name: chunk_id - type: str - - name: size - type: int - returns: - type: dict diff --git a/examples/epigenomics/agents/index_agent.py b/examples/epigenomics/agents/index_agent.py deleted file mode 100644 index 6faa984..0000000 --- a/examples/epigenomics/agents/index_agent.py +++ /dev/null @@ -1,30 +0,0 @@ -# Index Agent -# -# Final stage (mirrors Epigenomics' index-build stage): produces the -# workflow's terminal artifact from the merged, deduplicated digest. -# -# Resource profile: light CPU, single call per request. - -import hashlib - - -class IndexAgent(object): - def __init__(self): - self.tools = [self.build_index] - - def build_index(self, merged_digest: str, total_size: int) -> dict: - """Build the final index from the merged digest.""" - index_digest = self._cpu_work(merged_digest, max(1, total_size // 4)) - return {"index_digest": index_digest, "total_size": total_size} - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = IndexAgent() - print(agent.build_index("deadbeef", 65536)) diff --git a/examples/epigenomics/agents/index_agent.yaml b/examples/epigenomics/agents/index_agent.yaml deleted file mode 100644 index 3424c44..0000000 --- a/examples/epigenomics/agents/index_agent.yaml +++ /dev/null @@ -1,12 +0,0 @@ -agent: - name: IndexAgent - functions: - - name: build_index - description: Build the final index from the merged digest. - arguments: - - name: merged_digest - type: str - - name: total_size - type: int - returns: - type: dict diff --git a/examples/epigenomics/agents/map_agent.py b/examples/epigenomics/agents/map_agent.py deleted file mode 100644 index b9db3ef..0000000 --- a/examples/epigenomics/agents/map_agent.py +++ /dev/null @@ -1,33 +0,0 @@ -# Map Agent -# -# Sequence-alignment stand-in (Epigenomics' map stage) -- by far the most -# CPU-expensive stage in the real workflow, so its per-byte cost multiplier -# here is set well above the other stages to match that shape. -# -# Resource profile: heavy CPU, high fan-out (one call per chunk). - -import hashlib - -COST_MULTIPLIER = 8 - - -class MapAgent(object): - def __init__(self): - self.tools = [self.align] - - def align(self, chunk_id: str, size: int, digest: str) -> dict: - """Align one filtered chunk, returning its post-alignment digest.""" - aligned = self._cpu_work(digest, size * COST_MULTIPLIER) - return {"chunk_id": chunk_id, "size": size, "digest": aligned} - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = MapAgent() - print(agent.align("chunk-0", 16384, "deadbeef")) diff --git a/examples/epigenomics/agents/map_agent.yaml b/examples/epigenomics/agents/map_agent.yaml deleted file mode 100644 index 66c1210..0000000 --- a/examples/epigenomics/agents/map_agent.yaml +++ /dev/null @@ -1,14 +0,0 @@ -agent: - name: MapAgent - functions: - - name: align - description: Align one filtered chunk, returning its post-alignment digest. - arguments: - - name: chunk_id - type: str - - name: size - type: int - - name: digest - type: str - returns: - type: dict diff --git a/examples/epigenomics/agents/sort_agent.py b/examples/epigenomics/agents/sort_agent.py deleted file mode 100644 index 2aa2df7..0000000 --- a/examples/epigenomics/agents/sort_agent.py +++ /dev/null @@ -1,32 +0,0 @@ -# Sort Agent -# -# Third per-chunk stage in the fan-out (mirrors Epigenomics' sort stage): -# orders one aligned chunk, returning an updated digest for the fan-in below. -# -# Resource profile: moderate CPU, high fan-out (one call per chunk). - -import hashlib - -COST_MULTIPLIER = 2 - - -class SortAgent(object): - def __init__(self): - self.tools = [self.sort] - - def sort(self, chunk_id: str, size: int, digest: str) -> dict: - """Sort one aligned chunk, returning its post-sort digest.""" - sorted_digest = self._cpu_work(digest, size * COST_MULTIPLIER) - return {"chunk_id": chunk_id, "size": size, "digest": sorted_digest} - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = SortAgent() - print(agent.sort("chunk-0", 16384, "deadbeef")) diff --git a/examples/epigenomics/agents/sort_agent.yaml b/examples/epigenomics/agents/sort_agent.yaml deleted file mode 100644 index 464dc85..0000000 --- a/examples/epigenomics/agents/sort_agent.yaml +++ /dev/null @@ -1,14 +0,0 @@ -agent: - name: SortAgent - functions: - - name: sort - description: Sort one aligned chunk, returning its post-sort digest. - arguments: - - name: chunk_id - type: str - - name: size - type: int - - name: digest - type: str - returns: - type: dict diff --git a/examples/epigenomics/agents/split_agent.py b/examples/epigenomics/agents/split_agent.py deleted file mode 100644 index 1b1a6be..0000000 --- a/examples/epigenomics/agents/split_agent.py +++ /dev/null @@ -1,27 +0,0 @@ -# Split Agent -# -# Entry stage of the pipeline (mirrors WfCommons' Epigenomics fastq-split -# stage): splits one logical input into num_chunks equal-sized chunks for the -# downstream fan-out. There's no real sequence file here -- each chunk's -# "size" just stands in for its data volume, which is what every downstream -# stage prices its synthetic CPU work off of. -# -# Resource profile: cheap CPU, single call per request. - - -class SplitAgent(object): - def __init__(self): - self.tools = [self.split] - - def split(self, input_size: int, num_chunks: int) -> dict: - """Split input_size bytes of data into num_chunks equal chunks.""" - chunk_size = max(1, input_size // num_chunks) - chunks = [ - {"chunk_id": f"chunk-{i}", "size": chunk_size} for i in range(num_chunks) - ] - return {"chunks": chunks} - - -if __name__ == "__main__": - agent = SplitAgent() - print(agent.split(65536, 4)) diff --git a/examples/epigenomics/agents/split_agent.yaml b/examples/epigenomics/agents/split_agent.yaml deleted file mode 100644 index cc64e8e..0000000 --- a/examples/epigenomics/agents/split_agent.yaml +++ /dev/null @@ -1,12 +0,0 @@ -agent: - name: SplitAgent - functions: - - name: split - description: Split input_size bytes of data into num_chunks equal chunks. - arguments: - - name: input_size - type: int - - name: num_chunks - type: int - returns: - type: dict diff --git a/examples/epigenomics/config/global_controller.yaml b/examples/epigenomics/config/global_controller.yaml deleted file mode 100644 index 6e8b4b0..0000000 --- a/examples/epigenomics/config/global_controller.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Global Controller Configuration — synthetic Epigenomics DAG, local provider -# Lists all agents and the workflow that Ventis manages. -# -# FilterAgent/MapAgent/SortAgent get 2 replicas each since the workflow fans -# out one call per chunk to them -- exercises multi-replica scheduling on a -# purely local, LLM-free run. - -agents: - - name: SplitAgent - replicas: 1 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/split_agent.py - provider: local - - - name: FilterAgent - replicas: 2 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/filter_agent.py - provider: local - - - name: MapAgent - replicas: 2 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/map_agent.py - provider: local - - - name: SortAgent - replicas: 2 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/sort_agent.py - provider: local - - - name: DedupAgent - replicas: 1 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/dedup_agent.py - provider: local - - - name: IndexAgent - replicas: 1 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/index_agent.py - provider: local - - - name: Workflow - replicas: 1 - type: workflow - redis_port: 6379 - api_port: 8080 - workflow_file: workflow/epigenomics_workflow.py - provider: local - -poll_interval: 5 - -redis: - host: localhost - port: 6379 - db: 0 diff --git a/examples/epigenomics/config/policy.yaml b/examples/epigenomics/config/policy.yaml deleted file mode 100644 index 2c91415..0000000 --- a/examples/epigenomics/config/policy.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Policy-Based Routing Rules — synthetic Epigenomics DAG -# Each rule defines a match condition (key-value pairs to check against -# request context) and an access list of allowed services. -# Rules are evaluated most-specific-first (most matching keys wins). -# An empty match ({}) acts as a default fallback. - -rules: - - match: - origin: admin - access: all - - - match: {} - access: - - Workflow - - SplitAgent - - FilterAgent - - MapAgent - - SortAgent - - DedupAgent - - IndexAgent diff --git a/examples/epigenomics/workflow/epigenomics_workflow.py b/examples/epigenomics/workflow/epigenomics_workflow.py deleted file mode 100644 index 00bcc2b..0000000 --- a/examples/epigenomics/workflow/epigenomics_workflow.py +++ /dev/null @@ -1,97 +0,0 @@ -# Epigenomics Workflow -# -# WfCommons-style synthetic Epigenomics DAG for local, LLM-free testing: -# 0. SplitAgent - split input_size bytes into num_chunks chunks (single call) -# 1. FilterAgent - per-chunk contaminant filter (fan-out) -# 2. MapAgent - per-chunk alignment, the heaviest stage (fan-out) -# 3. SortAgent - per-chunk sort (fan-out) -# 4. DedupAgent - merge + dedup every sorted chunk (fan-in barrier) -# 5. IndexAgent - build the final index from the merged digest (single call) -# -# Every stage does deterministic SHA-256 work sized off chunk byte counts -- -# no LLM calls, no external services -- so results are reproducible and the -# fan-out width scales with num_chunks. -# -# After running `ventis build` and `ventis deploy`: -# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' \ -# -d '{"input_size": 65536, "num_chunks": 4}' -# curl http://localhost:8080/status/ - -import sys -import os - -# These path inserts are needed when running inside a Docker container -# where all files are copied flat into /app/. -sys.path.insert(0, os.path.dirname(__file__)) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) - -import json - -from deploy import deploy -from agents.split_agent import SplitAgent -from agents.filter_agent import FilterAgent -from agents.map_agent import MapAgent -from agents.sort_agent import SortAgent -from agents.dedup_agent import DedupAgent -from agents.index_agent import IndexAgent - - -def main(input_size: int = 65536, num_chunks: int = 4): - split_agent = SplitAgent() - filter_agent = FilterAgent() - map_agent = MapAgent() - sort_agent = SortAgent() - dedup_agent = DedupAgent() - index_agent = IndexAgent() - - # Stage 0: single call, produces the chunk list the fan-out below runs over. - split = json.loads( - split_agent.split(input_size=input_size, num_chunks=num_chunks).value() - ) - chunks = split["chunks"] - - # Stage 1: fan out one filter call per chunk -- every call returns a Future - # immediately, so all chunks are dispatched before we block on any of them. - filter_futures = { - c["chunk_id"]: filter_agent.filter_contams(chunk_id=c["chunk_id"], size=c["size"]) - for c in chunks - } - filtered = {cid: json.loads(f.value()) for cid, f in filter_futures.items()} - - # Stage 2: fan out alignment -- the heaviest stage -- one call per chunk. - map_futures = { - cid: map_agent.align(chunk_id=cid, size=r["size"], digest=r["digest"]) - for cid, r in filtered.items() - } - mapped = {cid: json.loads(f.value()) for cid, f in map_futures.items()} - - # Stage 3: fan out sort, one call per chunk. - sort_futures = { - cid: sort_agent.sort(chunk_id=cid, size=r["size"], digest=r["digest"]) - for cid, r in mapped.items() - } - sorted_chunks = {cid: json.loads(f.value()) for cid, f in sort_futures.items()} - - # Stage 4: fan-in barrier -- dedup needs every sorted chunk before it can run. - merged = json.loads( - dedup_agent.merge_dedup(chunks=list(sorted_chunks.values())).value() - ) - - # Stage 5: build the final index from the merged digest. - index = json.loads( - index_agent.build_index( - merged_digest=merged["merged_digest"], total_size=merged["total_size"] - ).value() - ) - - return { - "input_size": input_size, - "num_chunks": num_chunks, - "merged_digest": merged["merged_digest"], - "n_chunks": merged["n_chunks"], - "index_digest": index["index_digest"], - } - - -deploy(main, port=8080) diff --git a/examples/joke_writer/.car/app/.env.example b/examples/joke_writer/.car/app/.env.example deleted file mode 100644 index b846149..0000000 --- a/examples/joke_writer/.car/app/.env.example +++ /dev/null @@ -1,20 +0,0 @@ -# Copy this to `.env` and fill in the token. `config/global_controller.yaml` -# points `env_file:` at that copy, and it reaches every container as -# `docker run --env-file`. -# -# Keep the real token out of THIS file. `.env.example` is the one exception to -# the build context's exclusion of `.env*`, so whatever is written here is baked -# into the image; `.env` itself never enters the build and never leaves the host. - -# A Bedrock API key -- the long-term kind generated in the console, or a -# short-term one. botocore matches this exact name against bedrock-runtime's -# signingName (`bedrock`) and switches the client from SigV4 to bearer auth by -# itself, which is why neither joke_writer.py nor ventis/llm/bedrock.py mentions -# it. An IAM access key works too: drop AWS_ACCESS_KEY_ID and -# AWS_SECRET_ACCESS_KEY in instead and the same client signs with SigV4. -AWS_BEARER_TOKEN_BEDROCK= - -# Neither is a secret, and both have defaults in joke_writer.py -- they are here -# to name what the source reads. -BEDROCK_MODEL_ID=meta.llama3-8b-instruct-v1:0 -AWS_REGION=us-east-1 diff --git a/examples/joke_writer/.car/app/LICENSE b/examples/joke_writer/.car/app/LICENSE deleted file mode 100644 index 5600729..0000000 --- a/examples/joke_writer/.car/app/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 LangChain, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/examples/joke_writer/.car/app/README.md b/examples/joke_writer/.car/app/README.md deleted file mode 100644 index 930a410..0000000 --- a/examples/joke_writer/.car/app/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# Joke Writer - -A LangGraph map-reduce, ported to Ventis. Derived from -[langchain-ai/langchain-academy](https://github.com/langchain-ai/langchain-academy) -at `fa15bec` (`module-4/studio/map_reduce.py`, MIT — see `LICENSE`). - -Unlike the other targets in `examples/`, **the source here is not unmodified**. -`map_reduce.py` built a `ChatOpenAI` at module scope, and at the time of the port -an agent container had no way to carry an `OPENAI_API_KEY` — the port was blocked -at the credential wall until the model call was rewritten onto Bedrock. That wall -is gone now: `env_file` puts any key in the container. The Bedrock rewrite stayed -anyway, and [What the port cost](#what-the-port-cost) is honest about what that -means. - -## Overview - -Given a topic, the graph splits it into sub-topics, writes one joke per -sub-topic in parallel, then picks the best of them. - -1. `generate_topics` — one LLM call, turns the topic into three sub-topics, - validated into `Subjects`. -2. `generate_joke` — one LLM call per sub-topic. `continue_to_jokes` emits a - `Send` per subject, so this node runs N times per request with no shared - state between the runs. `jokes` is an `Annotated[list, operator.add]`, which - is how the N results merge back into one state. -3. `best_joke` — one LLM call over every joke, returns the winner by index. - -``` - START - | - generate_topics 1 call - | - continue_to_jokes Send x N - / | \ - joke joke joke N calls, no shared state - \ | / - best_joke 1 call - | - END -``` - -### Why this one - -It is the smallest project in reach whose control flow does something a single -process cannot: `Send` fans out to N independent calls per request. Everything -else about it is deliberately boring — four packages, no tools, no external -service, one API key. - -## The port - -| File | What it holds | -| --- | --- | -| `joke_writer.py` | The source. Three prompts, two schemas, three nodes, and the graph — still compiled, never executed under Ventis. | -| `agents/joke_agent.py` | `JokeAgent`. Three methods, each calling the source's node with the node's own state dict. Imports `joke_writer`; restates nothing. | -| `agents/joke_agent.yaml` | The three nodes declared as three functions on one agent. | -| `workflow/joke_workflow.py` | Where the graph went — the edges, the `Send` fan-out and the `operator.add` reducer, re-expressed as ordinary Python. | -| `config/global_controller.yaml` | `JokeAgent` at `replicas: 3`, plus the workflow. | -| `config/policy.yaml` | Default-allow for the two services. Not optional — a missing file kills `ventis deploy`. | - -Two decisions worth naming: - -**One agent, not three.** `generate_topics` and `best_joke` run once per request -and have no resource profile of their own. Splitting them out would buy two more -images and two more Redis round trips. What is hoisted is the fan-out, and that -is a workflow concern. - -**The graph is not the port.** `StateGraph`, `Send` and the `Annotated[list, -operator.add]` reducer are control flow owned by the LangGraph runtime, and -Ventis has no runtime to execute them. The workflow dispatches N -`generate_joke` calls across the three replicas and concatenates the results -itself. Every call is dispatched before any is resolved — `.value()` blocks, so -fusing the two lines into one comprehension would silently serialize the fan-out -and remove the reason to be on Ventis at all. - -## What the port cost - -This is no longer upstream's model stack. `ChatOpenAI` and -`with_structured_output` are gone; `ventis.llm.bedrock.call_bedrock` is the raw -converse API, so each node asks for JSON in its prompt and validates the reply -through the same pydantic schema upstream used. `_extract_json` exists only -because `with_structured_output` used to do that work. - -That rewrite is not something the `porting-to-ventis` skill should do on a -user's project — it is the credential wall, and the skill's instruction is to -report it. It was done here deliberately, so that this example is one that -actually deploys. - -**It would not be necessary today.** The rewrite bought one thing: boto3 builds -no client at import, so the agent could be *loaded* with no secret in the -container, back when `_launch_locally` passed five `-e` flags and all five were -`VENTIS_*`. `env_file` removes that constraint — an `OPENAI_API_KEY` now reaches -a container as readily as a Bedrock one, and upstream's `ChatOpenAI` at module -scope would import fine. What the rewrite still buys is narrower: a module-scope -client turns a missing key into `"No agent loaded"`, while a per-call one turns -it into a real error on `/status`. Worth knowing, not worth a rewrite. - -The example stays on Bedrock because it is the model call that has been end-to-end -verified here, and because `ventis/llm/bedrock.py` is where Ventis writes per-call -token telemetry onto the future. - -## Running it - -Copy `.env.example` to `.env` and put a Bedrock API key in it: - -```shell -cp .env.example .env -$EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... -``` - -`config/global_controller.yaml` points `env_file:` at that file, and every -container gets it as `docker run --env-file`. Nothing in this project reads the -variable: botocore matches the name against `bedrock-runtime`'s signingName and -switches the client from SigV4 to bearer auth on its own, so -`ventis/llm/bedrock.py` still builds a plain `boto3.client("bedrock-runtime")`. -An IAM access key instead of the bearer token works the same way. - -`.env` is gitignored and excluded from the build context — the key is in the -container's environment and not in the image. Deploy checks the path before it -launches anything, so a missing `.env` is one error line rather than three -replicas that come up and fail every request. - -```shell -ventis build -ventis deploy -``` - -```shell -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query": "animals"}' -curl http://localhost:8080/status/ -``` - -```json -{"request_id": "cb6cb62d...", "status": "done", "result": { - "topic": "animals", - "subjects": ["Wildlife Conservation", "Animal Behavior", "Endangered Species"], - "jokes": ["...", "...", "..."], - "best_selected_joke": "Why did the chimpanzee go to the doctor? Because it was going bananas!" -}} -``` - -`BEDROCK_MODEL_ID` and `AWS_REGION` (see `.env.example`) have defaults in -`joke_writer.py`; neither is a secret. The region has to match the one the key -was issued for. - -### Running the source outside Ventis - -`joke_writer.py` imports `ventis.llm.bedrock` first and falls back to the flat -`bedrock` copy an agent image gets, so the compiled graph still runs on its own -from a checkout of this repo: - -```shell -pip install -e ../.. # the ventis package -pip install langgraph pydantic typing_extensions boto3 -``` - -```python -from joke_writer import graph - -graph.invoke({"topic": "animals"}) -``` - -## Provenance - -Taken from `module-4/studio/`, which holds four unrelated graphs sharing one -directory. Only `map_reduce.py` and its license are here. - -| Left behind | Why | -| --- | --- | -| `parallelization.py`, `research_assistant.py`, `sub_graphs.py` | Other graphs in the same studio directory. The first two also need a Tavily key and Wikipedia. | -| `langgraph.json` | Registers all four graphs and points at `./.env`; a trimmed copy would only be useful for LangGraph Studio. | -| The module-4 notebooks | Teaching material for the same code. | -| `OPENAI_API_KEY`, `TAVILY_API_KEY` in `.env.example` | The first belongs to a model call that is no longer here; the second to the two graphs that are not here. | - -Nothing was added at the project root: there is no `pyproject.toml`, `setup.py` -or `requirements.txt`, exactly as upstream has none for module-4. That is why -`config/global_controller.yaml` has to declare `requirements:` by hand. diff --git a/examples/joke_writer/.car/app/joke_workflow.py b/examples/joke_writer/.car/app/joke_workflow.py deleted file mode 100644 index 76e4027..0000000 --- a/examples/joke_writer/.car/app/joke_workflow.py +++ /dev/null @@ -1,39 +0,0 @@ -r"""CanyonOS Core workflow for the map-reduce joke writer. - -This file is where the graph went. `generate_topics -> continue_to_jokes -> -generate_joke x N -> best_joke` is not a compiled StateGraph any more; it is the -three statements below, and the `Send` fan-out is N calls dispatched across -JokeAgent's replicas. - - curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query": "animals"}' - curl http://localhost:8080/status/ -""" - -import json - -from deploy import deploy -from joke_writer import JokeAgent - - -def main(query): - """Route: POST /main {"query": ""}""" - agent = JokeAgent() - - subjects = json.loads(agent.generate_topics(topic=query).value())["subjects"] - - futures = [agent.generate_joke(subject=s) for s in subjects] - written = [json.loads(f.value()) for f in futures] - written_jokes = [joke for result in written for joke in result["jokes"]] - - best = json.loads(agent.best_joke(topic=query, jokes=written_jokes).value()) - - return { - "topic": query, - "subjects": subjects, - "jokes": written_jokes, - "best_selected_joke": best["best_selected_joke"], - } - - -deploy(main, port=8080) diff --git a/examples/joke_writer/.car/app/joke_writer.py b/examples/joke_writer/.car/app/joke_writer.py deleted file mode 100644 index 9f96eb5..0000000 --- a/examples/joke_writer/.car/app/joke_writer.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Map-reduce joke writer. - -Derived from langchain-ai/langchain-academy `module-4/studio/map_reduce.py` -(MIT, see LICENSE). The graph shape, the three prompts and the two schemas are -upstream's. The model call is not: upstream builds a `ChatOpenAI` at module -scope, and when this was ported nothing could carry an OPENAI_API_KEY into an -agent container. Bedrock reaches the model through boto3, which builds no client -at import, so the same code loaded with no secret injected. - -`env_file` has since removed that constraint -- the key now travels to the -container in a .env and botocore reads AWS_BEARER_TOKEN_BEDROCK on its own. The -rewrite stayed regardless; README.md says what that costs. - -`with_structured_output` went with it. `call_bedrock` is the raw converse API, so -each node asks for JSON in the prompt and validates the reply through the same -pydantic schema upstream used. -""" - -import json -import operator -import os -import re -from typing import Annotated - -from typing_extensions import TypedDict - -from pydantic import BaseModel, ValidationError - -from langgraph.constants import Send -from langgraph.graph import END, StateGraph, START - -# Ventis copies bedrock.py flat into every agent image; the package path is for -# running this module outside a container. -try: - from ventis.llm.bedrock import call_bedrock -except ImportError: - from bedrock import call_bedrock - -# Prompts we will use. Upstream's, plus the JSON instruction that -# `with_structured_output` used to add on our behalf. -subjects_prompt = """Generate a list of 3 sub-topics that are all related to this overall topic: {topic}. -Respond with JSON only, no prose: {{"subjects": ["...", "...", "..."]}}""" -joke_prompt = """Generate a joke about {subject}. -Respond with JSON only, no prose: {{"joke": "..."}}""" -best_joke_prompt = """Below are a bunch of jokes about {topic}. Select the best one! Return the ID of the best one, starting 0 as the ID for the first joke. Jokes: \n\n {jokes} -Respond with JSON only, no prose: {{"id": 0}}""" - -# LLM. Both are read once at import; the container gets them from its -# environment, and neither is a secret. -MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0") -REGION = os.environ.get("AWS_REGION", "us-east-1") - - -def _extract_json(text): - """Pull the first JSON object out of a model reply. - - Even told to answer with JSON only, a model wraps it in a ```json fence or - prefaces it with a sentence. Upstream never needed this because - `with_structured_output` handled it; the converse API does not. - """ - text = re.sub(r"^\s*```(?:json)?|```\s*$", "", text.strip(), flags=re.MULTILINE) - try: - return json.loads(text) - except json.JSONDecodeError: - pass - # Fall back to the outermost braced span. - match = re.search(r"\{.*\}", text, flags=re.DOTALL) - if not match: - raise ValueError(f"joke_writer: no JSON in model output: {text!r}") - return json.loads(match.group(0)) - - -def _ask(prompt, schema, max_tokens): - """One converse() call, validated into `schema`. - - Raising on a bad reply is deliberate. A node that returned a default would - put a plausible-looking wrong answer into the state, and the reduce step - downstream indexes into the jokes list by an id the model chose -- a silent - default there picks the wrong joke instead of failing. - """ - response = call_bedrock( - model_id=MODEL_ID, - messages=[{"role": "user", "content": [{"text": prompt}]}], - inference_config={"maxTokens": max_tokens, "temperature": 0.0}, - region=REGION, - ) - text = response["output"]["message"]["content"][0]["text"] - if not text: - raise ValueError("joke_writer: LLM returned no output.") - try: - return schema(**_extract_json(text)) - except (ValidationError, TypeError) as exc: - raise ValueError( - f"joke_writer: {schema.__name__} not satisfied by model output: {text!r}" - ) from exc - - -# Define the state -class Subjects(BaseModel): - subjects: list[str] - -class BestJoke(BaseModel): - id: int - -class OverallState(TypedDict): - topic: str - subjects: list - jokes: Annotated[list, operator.add] - best_selected_joke: str - -def generate_topics(state: OverallState): - prompt = subjects_prompt.format(topic=state["topic"]) - response = _ask(prompt, Subjects, max_tokens=300) - return {"subjects": response.subjects} - -class JokeState(TypedDict): - subject: str - -class Joke(BaseModel): - joke: str - -def generate_joke(state: JokeState): - prompt = joke_prompt.format(subject=state["subject"]) - response = _ask(prompt, Joke, max_tokens=300) - return {"jokes": [response.joke]} - -def best_joke(state: OverallState): - jokes = "\n\n".join(state["jokes"]) - prompt = best_joke_prompt.format(topic=state["topic"], jokes=jokes) - response = _ask(prompt, BestJoke, max_tokens=100) - if not 0 <= response.id < len(state["jokes"]): - raise ValueError( - f"joke_writer: model chose joke {response.id} of {len(state['jokes'])}." - ) - return {"best_selected_joke": state["jokes"][response.id]} - -def continue_to_jokes(state: OverallState): - return [Send("generate_joke", {"subject": s}) for s in state["subjects"]] - -# Construct the graph: here we put everything together to construct our graph -graph_builder = StateGraph(OverallState) -graph_builder.add_node("generate_topics", generate_topics) -graph_builder.add_node("generate_joke", generate_joke) -graph_builder.add_node("best_joke", best_joke) -graph_builder.add_edge(START, "generate_topics") -graph_builder.add_conditional_edges("generate_topics", continue_to_jokes, ["generate_joke"]) -graph_builder.add_edge("generate_joke", "best_joke") -graph_builder.add_edge("best_joke", END) - -# Compile the graph -graph = graph_builder.compile() - - -class JokeAgent(object): - """The graph's nodes, exposed under the class name `agent.name` declares.""" - - def generate_topics(self, topic: str) -> dict: - return generate_topics({"topic": topic}) - - def generate_joke(self, subject: str) -> dict: - return generate_joke({"subject": subject}) - - def best_joke(self, topic: str, jokes: list) -> dict: - return best_joke({"topic": topic, "jokes": jokes}) diff --git a/examples/joke_writer/.car/config/global_controller.yaml b/examples/joke_writer/.car/config/global_controller.yaml deleted file mode 100644 index 8ed54ab..0000000 --- a/examples/joke_writer/.car/config/global_controller.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Deployment manifest for the map-reduce joke writer. -# -# `entrypoint` is the copied source itself: the adapter is appended to the -# bottom of joke_writer.py, so the module the agent needs is the one the class -# already lives in. - -agents: - - name: JokeAgent - # The fan-out. `generate_joke` is stateless, so the controller picks a - # replica at random per call and the workflow's N dispatched calls spread - # across these three. - entrypoint: joke_writer.py - provider: local - replicas: 3 - redis_port: 6379 - resources: - cpu: 1 - memory: 1024 - requirements: - - langgraph - - pydantic - - typing_extensions - - - name: Workflow - type: workflow - workflow_file: joke_workflow.py - api_port: 8080 - provider: local - replicas: 1 - redis_port: 6379 - -poll_interval: 5 - -redis: - host: localhost - port: 6379 - db: 0 - -otel: - # The dashboard api's own OTLP ingest. Must be the full url including the - # path: the http exporter uses an explicitly-passed endpoint verbatim and - # only appends /v1/traces when reading OTEL_EXPORTER_OTLP_ENDPOINT. - destinations: - - name: local - protocol: http - endpoint: http://host.docker.internal:3000/v1/traces - headers: {} - -# Relative to the application root (the directory `ventis` runs from), not -# `.car`. .env is gitignored and excluded from the build context; -# .env.example names what belongs in it. -env_file: .env diff --git a/examples/joke_writer/.car/config/joke_agent.yaml b/examples/joke_writer/.car/config/joke_agent.yaml deleted file mode 100644 index a5bc13f..0000000 --- a/examples/joke_writer/.car/config/joke_agent.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# The graph's three nodes, exposed as three methods on one agent. -# -# One agent, not three. `generate_topics` and `best_joke` run once per request -# and have no resource profile of their own; splitting them out would add two -# images, two dependency trees and a Redis round trip to buy nothing. What is -# hoisted is the `Send` fan-out, and that is a workflow concern. - -agent: - name: JokeAgent - functions: - - name: generate_topics - description: Split a topic into three related sub-topics. - arguments: - - name: topic - type: str - returns: - type: dict - - - name: generate_joke - description: Write one joke about one subject. - arguments: - - name: subject - type: str - returns: - type: dict - - - name: best_joke - description: Pick the best joke out of the ones written for a topic. - arguments: - - name: topic - type: str - - name: jokes - type: list - returns: - type: dict diff --git a/examples/joke_writer/.env.example b/examples/joke_writer/.env.example deleted file mode 100644 index b846149..0000000 --- a/examples/joke_writer/.env.example +++ /dev/null @@ -1,20 +0,0 @@ -# Copy this to `.env` and fill in the token. `config/global_controller.yaml` -# points `env_file:` at that copy, and it reaches every container as -# `docker run --env-file`. -# -# Keep the real token out of THIS file. `.env.example` is the one exception to -# the build context's exclusion of `.env*`, so whatever is written here is baked -# into the image; `.env` itself never enters the build and never leaves the host. - -# A Bedrock API key -- the long-term kind generated in the console, or a -# short-term one. botocore matches this exact name against bedrock-runtime's -# signingName (`bedrock`) and switches the client from SigV4 to bearer auth by -# itself, which is why neither joke_writer.py nor ventis/llm/bedrock.py mentions -# it. An IAM access key works too: drop AWS_ACCESS_KEY_ID and -# AWS_SECRET_ACCESS_KEY in instead and the same client signs with SigV4. -AWS_BEARER_TOKEN_BEDROCK= - -# Neither is a secret, and both have defaults in joke_writer.py -- they are here -# to name what the source reads. -BEDROCK_MODEL_ID=meta.llama3-8b-instruct-v1:0 -AWS_REGION=us-east-1 diff --git a/examples/joke_writer/LICENSE b/examples/joke_writer/LICENSE deleted file mode 100644 index 5600729..0000000 --- a/examples/joke_writer/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 LangChain, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/examples/joke_writer/README.md b/examples/joke_writer/README.md deleted file mode 100644 index 930a410..0000000 --- a/examples/joke_writer/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# Joke Writer - -A LangGraph map-reduce, ported to Ventis. Derived from -[langchain-ai/langchain-academy](https://github.com/langchain-ai/langchain-academy) -at `fa15bec` (`module-4/studio/map_reduce.py`, MIT — see `LICENSE`). - -Unlike the other targets in `examples/`, **the source here is not unmodified**. -`map_reduce.py` built a `ChatOpenAI` at module scope, and at the time of the port -an agent container had no way to carry an `OPENAI_API_KEY` — the port was blocked -at the credential wall until the model call was rewritten onto Bedrock. That wall -is gone now: `env_file` puts any key in the container. The Bedrock rewrite stayed -anyway, and [What the port cost](#what-the-port-cost) is honest about what that -means. - -## Overview - -Given a topic, the graph splits it into sub-topics, writes one joke per -sub-topic in parallel, then picks the best of them. - -1. `generate_topics` — one LLM call, turns the topic into three sub-topics, - validated into `Subjects`. -2. `generate_joke` — one LLM call per sub-topic. `continue_to_jokes` emits a - `Send` per subject, so this node runs N times per request with no shared - state between the runs. `jokes` is an `Annotated[list, operator.add]`, which - is how the N results merge back into one state. -3. `best_joke` — one LLM call over every joke, returns the winner by index. - -``` - START - | - generate_topics 1 call - | - continue_to_jokes Send x N - / | \ - joke joke joke N calls, no shared state - \ | / - best_joke 1 call - | - END -``` - -### Why this one - -It is the smallest project in reach whose control flow does something a single -process cannot: `Send` fans out to N independent calls per request. Everything -else about it is deliberately boring — four packages, no tools, no external -service, one API key. - -## The port - -| File | What it holds | -| --- | --- | -| `joke_writer.py` | The source. Three prompts, two schemas, three nodes, and the graph — still compiled, never executed under Ventis. | -| `agents/joke_agent.py` | `JokeAgent`. Three methods, each calling the source's node with the node's own state dict. Imports `joke_writer`; restates nothing. | -| `agents/joke_agent.yaml` | The three nodes declared as three functions on one agent. | -| `workflow/joke_workflow.py` | Where the graph went — the edges, the `Send` fan-out and the `operator.add` reducer, re-expressed as ordinary Python. | -| `config/global_controller.yaml` | `JokeAgent` at `replicas: 3`, plus the workflow. | -| `config/policy.yaml` | Default-allow for the two services. Not optional — a missing file kills `ventis deploy`. | - -Two decisions worth naming: - -**One agent, not three.** `generate_topics` and `best_joke` run once per request -and have no resource profile of their own. Splitting them out would buy two more -images and two more Redis round trips. What is hoisted is the fan-out, and that -is a workflow concern. - -**The graph is not the port.** `StateGraph`, `Send` and the `Annotated[list, -operator.add]` reducer are control flow owned by the LangGraph runtime, and -Ventis has no runtime to execute them. The workflow dispatches N -`generate_joke` calls across the three replicas and concatenates the results -itself. Every call is dispatched before any is resolved — `.value()` blocks, so -fusing the two lines into one comprehension would silently serialize the fan-out -and remove the reason to be on Ventis at all. - -## What the port cost - -This is no longer upstream's model stack. `ChatOpenAI` and -`with_structured_output` are gone; `ventis.llm.bedrock.call_bedrock` is the raw -converse API, so each node asks for JSON in its prompt and validates the reply -through the same pydantic schema upstream used. `_extract_json` exists only -because `with_structured_output` used to do that work. - -That rewrite is not something the `porting-to-ventis` skill should do on a -user's project — it is the credential wall, and the skill's instruction is to -report it. It was done here deliberately, so that this example is one that -actually deploys. - -**It would not be necessary today.** The rewrite bought one thing: boto3 builds -no client at import, so the agent could be *loaded* with no secret in the -container, back when `_launch_locally` passed five `-e` flags and all five were -`VENTIS_*`. `env_file` removes that constraint — an `OPENAI_API_KEY` now reaches -a container as readily as a Bedrock one, and upstream's `ChatOpenAI` at module -scope would import fine. What the rewrite still buys is narrower: a module-scope -client turns a missing key into `"No agent loaded"`, while a per-call one turns -it into a real error on `/status`. Worth knowing, not worth a rewrite. - -The example stays on Bedrock because it is the model call that has been end-to-end -verified here, and because `ventis/llm/bedrock.py` is where Ventis writes per-call -token telemetry onto the future. - -## Running it - -Copy `.env.example` to `.env` and put a Bedrock API key in it: - -```shell -cp .env.example .env -$EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... -``` - -`config/global_controller.yaml` points `env_file:` at that file, and every -container gets it as `docker run --env-file`. Nothing in this project reads the -variable: botocore matches the name against `bedrock-runtime`'s signingName and -switches the client from SigV4 to bearer auth on its own, so -`ventis/llm/bedrock.py` still builds a plain `boto3.client("bedrock-runtime")`. -An IAM access key instead of the bearer token works the same way. - -`.env` is gitignored and excluded from the build context — the key is in the -container's environment and not in the image. Deploy checks the path before it -launches anything, so a missing `.env` is one error line rather than three -replicas that come up and fail every request. - -```shell -ventis build -ventis deploy -``` - -```shell -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query": "animals"}' -curl http://localhost:8080/status/ -``` - -```json -{"request_id": "cb6cb62d...", "status": "done", "result": { - "topic": "animals", - "subjects": ["Wildlife Conservation", "Animal Behavior", "Endangered Species"], - "jokes": ["...", "...", "..."], - "best_selected_joke": "Why did the chimpanzee go to the doctor? Because it was going bananas!" -}} -``` - -`BEDROCK_MODEL_ID` and `AWS_REGION` (see `.env.example`) have defaults in -`joke_writer.py`; neither is a secret. The region has to match the one the key -was issued for. - -### Running the source outside Ventis - -`joke_writer.py` imports `ventis.llm.bedrock` first and falls back to the flat -`bedrock` copy an agent image gets, so the compiled graph still runs on its own -from a checkout of this repo: - -```shell -pip install -e ../.. # the ventis package -pip install langgraph pydantic typing_extensions boto3 -``` - -```python -from joke_writer import graph - -graph.invoke({"topic": "animals"}) -``` - -## Provenance - -Taken from `module-4/studio/`, which holds four unrelated graphs sharing one -directory. Only `map_reduce.py` and its license are here. - -| Left behind | Why | -| --- | --- | -| `parallelization.py`, `research_assistant.py`, `sub_graphs.py` | Other graphs in the same studio directory. The first two also need a Tavily key and Wikipedia. | -| `langgraph.json` | Registers all four graphs and points at `./.env`; a trimmed copy would only be useful for LangGraph Studio. | -| The module-4 notebooks | Teaching material for the same code. | -| `OPENAI_API_KEY`, `TAVILY_API_KEY` in `.env.example` | The first belongs to a model call that is no longer here; the second to the two graphs that are not here. | - -Nothing was added at the project root: there is no `pyproject.toml`, `setup.py` -or `requirements.txt`, exactly as upstream has none for module-4. That is why -`config/global_controller.yaml` has to declare `requirements:` by hand. diff --git a/examples/joke_writer/joke_writer.py b/examples/joke_writer/joke_writer.py deleted file mode 100644 index 3ad49d5..0000000 --- a/examples/joke_writer/joke_writer.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Map-reduce joke writer. - -Derived from langchain-ai/langchain-academy `module-4/studio/map_reduce.py` -(MIT, see LICENSE). The graph shape, the three prompts and the two schemas are -upstream's. The model call is not: upstream builds a `ChatOpenAI` at module -scope, and when this was ported nothing could carry an OPENAI_API_KEY into an -agent container. Bedrock reaches the model through boto3, which builds no client -at import, so the same code loaded with no secret injected. - -`env_file` has since removed that constraint -- the key now travels to the -container in a .env and botocore reads AWS_BEARER_TOKEN_BEDROCK on its own. The -rewrite stayed regardless; README.md says what that costs. - -`with_structured_output` went with it. `call_bedrock` is the raw converse API, so -each node asks for JSON in the prompt and validates the reply through the same -pydantic schema upstream used. -""" - -import json -import operator -import os -import re -from typing import Annotated - -from typing_extensions import TypedDict - -from pydantic import BaseModel, ValidationError - -from langgraph.constants import Send -from langgraph.graph import END, StateGraph, START - -# Ventis copies bedrock.py flat into every agent image; the package path is for -# running this module outside a container. -try: - from ventis.llm.bedrock import call_bedrock -except ImportError: - from bedrock import call_bedrock - -# Prompts we will use. Upstream's, plus the JSON instruction that -# `with_structured_output` used to add on our behalf. -subjects_prompt = """Generate a list of 3 sub-topics that are all related to this overall topic: {topic}. -Respond with JSON only, no prose: {{"subjects": ["...", "...", "..."]}}""" -joke_prompt = """Generate a joke about {subject}. -Respond with JSON only, no prose: {{"joke": "..."}}""" -best_joke_prompt = """Below are a bunch of jokes about {topic}. Select the best one! Return the ID of the best one, starting 0 as the ID for the first joke. Jokes: \n\n {jokes} -Respond with JSON only, no prose: {{"id": 0}}""" - -# LLM. Both are read once at import; the container gets them from its -# environment, and neither is a secret. -MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0") -REGION = os.environ.get("AWS_REGION", "us-east-1") - - -def _extract_json(text): - """Pull the first JSON object out of a model reply. - - Even told to answer with JSON only, a model wraps it in a ```json fence or - prefaces it with a sentence. Upstream never needed this because - `with_structured_output` handled it; the converse API does not. - """ - text = re.sub(r"^\s*```(?:json)?|```\s*$", "", text.strip(), flags=re.MULTILINE) - try: - return json.loads(text) - except json.JSONDecodeError: - pass - # Fall back to the outermost braced span. - match = re.search(r"\{.*\}", text, flags=re.DOTALL) - if not match: - raise ValueError(f"joke_writer: no JSON in model output: {text!r}") - return json.loads(match.group(0)) - - -def _ask(prompt, schema, max_tokens): - """One converse() call, validated into `schema`. - - Raising on a bad reply is deliberate. A node that returned a default would - put a plausible-looking wrong answer into the state, and the reduce step - downstream indexes into the jokes list by an id the model chose -- a silent - default there picks the wrong joke instead of failing. - """ - response = call_bedrock( - model_id=MODEL_ID, - messages=[{"role": "user", "content": [{"text": prompt}]}], - inference_config={"maxTokens": max_tokens, "temperature": 0.0}, - region=REGION, - ) - text = response["output"]["message"]["content"][0]["text"] - if not text: - raise ValueError("joke_writer: LLM returned no output.") - try: - return schema(**_extract_json(text)) - except (ValidationError, TypeError) as exc: - raise ValueError( - f"joke_writer: {schema.__name__} not satisfied by model output: {text!r}" - ) from exc - - -# Define the state -class Subjects(BaseModel): - subjects: list[str] - -class BestJoke(BaseModel): - id: int - -class OverallState(TypedDict): - topic: str - subjects: list - jokes: Annotated[list, operator.add] - best_selected_joke: str - -def generate_topics(state: OverallState): - prompt = subjects_prompt.format(topic=state["topic"]) - response = _ask(prompt, Subjects, max_tokens=300) - return {"subjects": response.subjects} - -class JokeState(TypedDict): - subject: str - -class Joke(BaseModel): - joke: str - -def generate_joke(state: JokeState): - prompt = joke_prompt.format(subject=state["subject"]) - response = _ask(prompt, Joke, max_tokens=300) - return {"jokes": [response.joke]} - -def best_joke(state: OverallState): - jokes = "\n\n".join(state["jokes"]) - prompt = best_joke_prompt.format(topic=state["topic"], jokes=jokes) - response = _ask(prompt, BestJoke, max_tokens=100) - if not 0 <= response.id < len(state["jokes"]): - raise ValueError( - f"joke_writer: model chose joke {response.id} of {len(state['jokes'])}." - ) - return {"best_selected_joke": state["jokes"][response.id]} - -def continue_to_jokes(state: OverallState): - return [Send("generate_joke", {"subject": s}) for s in state["subjects"]] - -# Construct the graph: here we put everything together to construct our graph -graph_builder = StateGraph(OverallState) -graph_builder.add_node("generate_topics", generate_topics) -graph_builder.add_node("generate_joke", generate_joke) -graph_builder.add_node("best_joke", best_joke) -graph_builder.add_edge(START, "generate_topics") -graph_builder.add_conditional_edges("generate_topics", continue_to_jokes, ["generate_joke"]) -graph_builder.add_edge("generate_joke", "best_joke") -graph_builder.add_edge("best_joke", END) - -# Compile the graph -graph = graph_builder.compile() diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md deleted file mode 100644 index fe6dd49..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md +++ /dev/null @@ -1,240 +0,0 @@ ---- -name: porting-to-canyonos-core -description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. -compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. ---- - -# Port an agent project to CanyonOS Core - -CanyonOS Core is the product name. Its compatibility executable and Python -package remain `ventis`; environment variables and Docker resources retain the -`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding -strings. Do not rename them. - -## Load references only when needed - -- Read [references/packaging.md](references/packaging.md) when a source import - does not resolve from `/app`, the source is nested, or packaging metadata is - involved. -- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target - includes `llm_proxy`. -- Read [references/ec2.md](references/ec2.md) only when any config entry uses - `provider: EC2`. -- Read [references/troubleshooting.md](references/troubleshooting.md) after a - failed build, image probe, deploy, or request. -- Read [references/runtime-contract.md](references/runtime-contract.md) when a - validator finding needs explanation or the runtime mechanism is unclear. - -## Goal: thin scaffolding beside untouched source - -```text -agents/.yaml one callable surface per service -agents/.py one thin adapter per service, when needed -workflow/_workflow.py HTTP entry point; calls deploy() -config/global_controller.yaml deployment manifest -config/policy.yaml optional access restriction -pyproject.toml conditional nested-import scaffolding - unchanged -``` - -The file count follows the deployment. A multi-agent port has one yaml/adapter -pair per service that is worth deploying separately. If a source class already -satisfies the runtime contract, point its config entry at that file and do not -copy it into an adapter. - -Everything the source already owns—prompts, tools, schemas, parsing, retries, -model clients, and node bodies—is imported. The port re-expresses only the -CanyonOS Core boundary and framework-owned orchestration. - -The port root is the existing repository root and the directory from which -`ventis build` runs. Write scaffolding there beside existing directories. If the -repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, -and `config/` beside it. Never move or copy the repository into a new `src/` -directory, and never create an outer wrapper merely for the port. - -## 1. Survey before writing - -Identify: - -1. The source entry point and callable input/output. -2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, - `Send`, `Command`, interrupts). -3. Runtime-injected services nodes read: stores, context, memory, sessions, or - callback managers. -4. Sync versus async boundaries. -5. Imports and declared runtime dependencies. -6. Model provider, credential names, streaming use, and optional `llm_proxy`. -7. Whether independent work fans out and benefits from separate replicas. -8. Whether source imports resolve from the project root that becomes `/app`. - -Run the validator once now. Its header detects capabilities directly from the -importable runtime rather than external development metadata: - -```bash -python /validate.py . -``` - -If config or agent yaml is malformed, the validator defers to `ventis build`. -Capability-gated findings say which runtime behavior is available. - -## 2. Choose service boundaries - -Start with one service. Split only when it creates independent parallel work or -a distinct resource/replica profile. - -- Keep a ReAct loop together; every turn needs shared message history. -- Hoist supervisor task lists and `Send`-style fan-out into the workflow. -- Do not create a one-replica service with no distinct resource profile merely - to mirror every source graph node. - -Rewrite framework-owned edges as ordinary Python. Import the connected node -functions unchanged. Construct runtime-injected service objects from source -configuration; do not invent models, dimensions, stores, or defaults silently. -Report any choice the source does not specify. - -## 3. Write declarations and adapters - -### Agent yaml - -Use one yaml per deployed service. Argument types are bare builtins only: -`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is -required by the generated stub. `returns.type` is documentation; use `dict` or -`list` to signal that workflow callers must `json.loads` the returned string. - -### Adapter - -The entrypoint exposes a module-level class named exactly `agent.name`. It -constructs with no arguments and its declared methods are synchronous. Read -configuration from the environment in `__init__`. Bridge source coroutines -inside a synchronous method with `asyncio.run(...)`. Serialize framework objects -with their own JSON-safe serializer before returning. - -Do not duplicate source prompts, tools, schemas, or model calls. Keep the source -provider and SDK. - -### Workflow - -Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. -Import generated stubs by yaml basename and agent class name: - -```python -from deploy import deploy -from agents. import -``` - -The deployment platform sends `{query: string}` to `/main`. Pack richer input -inside `query`; any additional workflow parameter has a default. - -Dispatch every remote call before resolving any future: - -```python -futures = [agent.work(item=item) for item in items] -results = [json.loads(future.value()) for future in futures] -``` - -Do not fuse dispatch and `.value()` in one comprehension; that silently -serializes fan-out. Do not add an `if __name__ == "__main__":` block: the -workflow is executed with `__name__ == "__main__"` in production. - -### Config - -For each service, keep these names aligned: - -```text -config entry name == yaml agent.name == entrypoint class name -``` - -Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a -list of distribution-name strings. Put `env_file` at config top level when the -runtime capability is available. Omit `policy.yaml` unless access must be -restricted; if present, give it a non-empty `rules` list. - -## Hard rules - -Capitalized **MUST** and **NEVER** are reserved for port-breaking or -source-integrity rules. The owner column states where each is decided. - -| ID | Rule | Owner | -|---|---|---| -| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | -| M2 | The class MUST construct with no arguments | V007 | -| M3 | yaml argument names MUST match Python parameter names | V008 | -| M4 | yaml argument types MUST be bare builtins | V010 | -| M5 | Declared adapter methods MUST be synchronous | V009 | -| M6 | Config names MUST match yaml agent names | build | -| M7 | Config names MUST not collide after lowercase normalization | build output | -| M8 | Local provider MUST be lowercase `local` | deploy preflight | -| M9 | `replicas` MUST be an integer | deploy preflight | -| M10 | `requirements` MUST be a list of strings | build | -| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | -| M12 | Workflow MUST NEVER contain a main guard | V017 | -| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | -| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | -| M15 | Workflow MUST import stubs from `agents.` | V023 | -| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | -| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | -| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | -| M19 | NEVER hardcode or bake a real credential into an image | W003 | -| M20 | NEVER edit or vendor the source tree | `git status` | -| M21 | NEVER swap the source LLM provider | review | -| M22 | NEVER silently move, drop, or reclassify source dependencies | review | -| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | -| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | - -## 4. Validate, build, and probe - -Run static preflight, then let the build own build-time validation: - -```bash -python /validate.py . -ventis build -c config/global_controller.yaml -``` - -A green build never imports the adapter. Probe each agent image in this order: - -```bash -# Runtime startup path - docker run --rm ventis- \ - python -c "import local_controller" - -# Agent load path; include --env-file when configured - docker run --rm --env-file ventis- \ - python -c "import importlib.util,sys; \ -s=importlib.util.spec_from_file_location('m','.py'); \ -m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ -m.();print('ok')" -``` - -Also probe the workflow image with `python -c "import local_controller"`; it has -its own dependency resolve and generated-stub imports. - -Then deploy, send a representative request, and poll its status: - -```bash -ventis deploy -c config/global_controller.yaml -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query":""}' -curl http://localhost:8080/status/ -``` - -A successful outer request with a source-level failure still proves the port -reached and returned the source behavior. Record the distinction. - -## 5. Clean up - -After collecting evidence, stop foreground deploy with Ctrl+C and wait for -controller cleanup. Remove exact leftovers if startup crashed. Then remove build -products and exact images from this config: - -```bash -ventis clean -docker image rm ventis- \ - ventis- - -test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container -docker ps -a --format '{{.Names}}' -``` - -`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it -does not remove containers or images. Keep port scaffolding, untouched source, -and requested logs or reports. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md deleted file mode 100644 index e06daa0..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md +++ /dev/null @@ -1,41 +0,0 @@ -# EC2 deployment - -Read this only when at least one config entry uses `provider: EC2`. - -## Configuration - -Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The -top-level `ec2` block supplies the runtime's required infrastructure and SSH -settings. Read the target checkout's deploy preflight and EC2 runtime before -writing the block; do not copy values from an example environment. - -Typical required categories are: - -- AMI and instance type -- region and subnet -- security groups -- SSH user and credentials accepted by the runtime - -`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof -that provisioning, SSH, image transfer, or remote container startup works. - -## Networking - -A remote container's `host.docker.internal` names its own EC2 Docker host. It -does not name the local controller machine. Databases, model proxies, and other -services must use addresses reachable from every selected host. - -The environment file may be copied temporarily to a remote host by runtimes that -expose the `env_file` capability. Confirm behavior from the capability probe and -target runtime rather than assuming local Docker semantics. - -## Probes and cleanup - -Run the same runtime and adapter probes against the exact image before remote -deployment. After deploy, verify the remote container logs; controller health -can be green even when agent loading failed. - -Stop foreground deploy normally so the controller can terminate recorded EC2 -instances. If provisioning or startup fails before an instance is recorded, -inspect the cloud provider directly and remove exact leaked resources. Never use -a broad cleanup command against unrelated instances. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md deleted file mode 100644 index f726bd7..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md +++ /dev/null @@ -1,64 +0,0 @@ -# LLM proxy integration - -Read this only when the target checkout contains `llm_proxy` or the deployment -explicitly routes model SDKs through it. - -## Preserve provider protocols - -The proxy redirects provider endpoints; it does not convert providers. Keep the -source SDK, model ID, request body, and response parsing unchanged. - -Configure only the provider variables the source uses: - -```dotenv -OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 -ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic -AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock -``` - -Some SDKs refuse to initialize without caller credentials. Give agent containers -non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS -credentials in the separate proxy process, not in the port's `env_file`. - -## Start locally - -The proxy defaults conflict with a typical deployment: host loopback is not -reachable from a container, and port 8080 is normally used by the workflow API. -Use a non-loopback bind and a different port: - -```bash -PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy -curl http://127.0.0.1:8081/healthz -``` - -Local CanyonOS Core containers resolve `host.docker.internal` through their -Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the -machine running `ventis deploy`. Distributed deployments need a reachable proxy -address or one proxy on each host. - -## Supported call shape - -The implementation buffers complete requests and responses: - -- OpenAI and Anthropic non-streaming HTTP calls are forwarded. -- Bedrock `invoke` is reissued through the proxy's boto3 client. -- OpenAI/Anthropic streaming is unsupported. -- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are - unsupported. - -Survey the source before selecting the proxy. Do not silently disable streaming; -report the unsupported behavior and stop. - -## Credential behavior - -- The OpenAI adapter removes caller authorization and inserts the proxy key. -- The Anthropic adapter removes caller key headers and inserts the proxy key. -- Botocore still signs requests sent to a custom endpoint, so a caller may need - placeholder AWS credentials even though the proxy reissues upstream with its - own identity. -- `/healthz` proves provider registration and Flask availability, not upstream - credential validity. - -OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return -JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed -with the upstream status and are not byte-for-byte passthrough. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md deleted file mode 100644 index 8520c4a..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md +++ /dev/null @@ -1,81 +0,0 @@ -# Packaging and import roots - -Read this reference when an adapter imports nested source code, the source uses a -`src/` layout, or V031 reports an import-root problem. - -## What `/app` can import - -CanyonOS Core preserves project-relative paths in the image and starts Python at -`/app`. Without an editable install, Python resolves names rooted there: - -- `/app/tools.py` as `import tools` -- `/app/pkg/__init__.py` as `import pkg` -- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace - directories without `__init__.py` - -It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become -an import root first. - -## Detect support, do not infer it from release history - -Run: - -```bash -python /validate.py . -``` - -Read the `editable_install` capability. If it is unavailable and the original -import cannot resolve from `/app`, report a runtime capability blocker and stop. -Do not add a `sys.path` hack or relocate source files. - -## Root metadata is the trigger - -When editable install is supported, only packaging metadata at the **port root** -triggers `pip install -e .`: - -```text -port-root/pyproject.toml detected -port-root/source/pyproject.toml ignored as an install trigger -``` - -A nested source repository may remain untouched. Add minimal root scaffolding -that points package discovery at the existing source package: - -```toml -[build-system] -requires = ["setuptools>=64"] -build-backend = "setuptools.build_meta" - -[project] -name = "canyonos-port" -version = "0.0.0" -dependencies = [] - -[tool.setuptools.packages.find] -where = ["source/src"] -include = ["pkg*"] -namespaces = true -``` - -Set `where` and `include` from the actual tree and original import spelling. Do -not reference a README or license from this wrapper metadata; file sweeps differ -by runtime capability and a missing referenced file makes the image build fail. - -## Dependencies in nested metadata - -A nested `pyproject.toml` is not installed merely because its Python files are -copied. Keep the source declaration unchanged and repeat its runtime -distributions in each relevant config entry's `requirements` list. This is -compatibility scaffolding, not permission to drop, move, or reclassify declared -dependencies. - -If source metadata is already at the port root, do not create a wrapper. Its -project dependencies participate in the same resolver as config requirements. -Report declared-but-unused toolchain dependencies and their image cost; let the -owner decide whether source metadata should change. - -## Validation boundary - -`ventis build` owns packaging syntax and installation errors. `validate.py` -checks only whether adapter imports appear to require a nested root that the -runtime will not expose. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md deleted file mode 100644 index 94f7401..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md +++ /dev/null @@ -1,187 +0,0 @@ -# CanyonOS Core runtime contract - -The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the -CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for -Docker resources. - -Read this reference when implementing an adapter or explaining a validator -finding. Runtime-dependent behavior is expressed as capabilities; run -`validate.py` against the target environment instead of inferring support from -release history. - -## Project root and discovery - -`ventis build` uses the current working directory as the project root. - -| Input | Discovery | -|---|---| -| `agents/*.yaml` | direct yaml glob under the project root | -| `config/global_controller.yaml` | default config, overridable with `-c` | -| workflow | `workflow_file` on a `type: workflow` entry | -| policy | `policy.yaml` beside the selected config file | -| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | - -The config name, yaml `agent.name`, and entrypoint class name form one binding: - -```text -config entry name == yaml agent.name == entrypoint class name -``` - -A missing match may skip an image while the command continues, so inspect build -output and generated image tags. - -## Agent yaml and generated stubs - -The consumed yaml shape is: - -```yaml -agent: - name: ExampleAgent - functions: - - name: work - arguments: - - name: query - type: str - returns: - type: dict -``` - -Argument annotations are generated from bare names without adding imports. Use -builtins. Generated methods have no defaults, so every declared argument is -required at the stub call site. `returns` does not control runtime conversion; -it documents whether workflow code should parse the returned string. - -Stub destinations differ by runtime capability and entrypoint layout. Workflow -code in this port convention imports the generated class from -`agents.`. The validator checks that import against declarations -before the workflow image starts. - -## Agent loading and execution - -The local controller effectively performs: - -```python -module = load(entrypoint) -agent_class = getattr(module, configured_name) -agent = agent_class() -result = getattr(agent, method_name)(**args) -``` - -Consequences: - -- The class is module-level and named exactly as configured. -- Construction takes no arguments. -- Declared methods accept yaml argument names as keyword arguments. -- Methods are synchronous; this path does not await a coroutine. -- Dicts and lists are JSON-encoded before entering Redis; other results become - strings. -- A remote Future's `.value()` returns text, not the original Python object. - -Agent import and construction exceptions are caught by the controller. A failed -agent may still advertise healthy because health is written independently of -successful agent loading. That is why image probes import both the runtime and -the entrypoint explicitly. - -## Workflow execution - -The workflow file is executed, not imported. Therefore: - -- module-level code runs at container startup; -- `__name__ == "__main__"`; -- `deploy()` blocks in the web server; -- the workflow function runs once per request; -- its function name determines the REST route exposed by the compatibility - runtime. - -The deployment platform additionally expects `/main` with a `{query: string}` -body. This platform constraint is stricter than the underlying transport. - -Each stub method returns a Future immediately. `.value()` blocks. Dispatching -and resolving inside one comprehension serializes work without raising an -error; dispatch all calls first, then resolve them. - -The workflow container also starts runtime controller code and has its own -package resolution. Probe it independently from agent images. - -## Build context and collisions - -The runtime copies project files while preserving relative paths, then writes -shared runtime modules, generated stubs, and entrypoints into the image. Later -writes can shadow project files. - -Avoid root project modules named like runtime files, including: - -```text -future.py -ventis_context.py -local_controller.py -local_controller_frontend.py -redis_client.py -grpc_options.py -bedrock.py -deploy.py -session_logging.py -workflow_launcher.py -``` - -Also avoid a yaml basename that shadows a different source module imported by an -adapter. The validator checks deterministic flat-name collisions. - -File sweep and editable-install behavior are runtime capabilities. For nested -imports, follow [packaging.md](packaging.md). - -## Dependencies and protobuf - -Agent and workflow images include a small runtime dependency set. Config -`requirements` adds source-specific distributions. A malformed requirements -value can be normalized away while image generation continues; missing imports -then surface only when the agent loads. - -The build compiles gRPC Python stubs on the host and copies them into images. -The image resolver does not necessarily know the generated-code version. A -source dependency that constrains protobuf below the host generator version can -produce a green image build that dies on: - -```text -import local_controller -``` - -Always run that probe before probing the entrypoint. Treat a generated-code / -runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter -source dependencies silently. - -## Credentials capability - -When `env_file` capability is available, the top-level config path is resolved -against the project root and passed at container start. Hidden env files are not -copied into images. Invalid paths are deploy-preflight errors. - -When the capability is unavailable, declaring `env_file` has no effect. If the -source needs credentials, report the capability blocker rather than hardcoding -or vendoring a secret. - -A source that constructs its client at import time works only when credentials -are already in the container environment. Image entrypoint probes therefore use -the same env file as deployment. - -For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). - -## Policy and provider behavior - -No policy file means unrestricted service access. If a policy exists, it needs a -non-empty rules list. Rules are evaluated by specificity and first match; -services excluded from the selected rule fail after request acceptance. - -Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior -and remote networking are covered in [ec2.md](ec2.md). - -## Cleanup boundary - -Stopping foreground deploy normally invokes controller cleanup for recorded -containers and Redis. Hard kills and failures before resource registration may -leave resources behind. - -`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and -`docker_container/`. It does not remove containers or images. Remove exact -leftovers explicitly and preserve source, port scaffolding, and requested -evidence. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md deleted file mode 100644 index 361acf9..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md +++ /dev/null @@ -1,60 +0,0 @@ -# Troubleshooting - -Read this after a failed build, image probe, deploy, or request. For mechanisms, -read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, -read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). - -## Build or deploy stops early - -| Symptom | Likely cause | -|---|---| -| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | -| Two services produce one image | Config names collide after lowercase normalization | -| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | -| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | -| Replica conversion `TypeError` | `replicas` is not an integer | -| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | -| Port or container name already in use | A previous deployment did not complete cleanup | - -## Container exits or serves nothing - -| Symptom | Likely cause | -|---|---| -| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | -| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | -| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | -| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | -| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | -| Third-party module is missing | Distribution is absent from source metadata and config requirements | -| Stub import raises `NameError` | yaml argument type is not a bare builtin | -| Source module behaves like an empty stub | A generated stub basename shadowed the source module | -| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | - -## Request is accepted, then fails - -| Symptom | Likely cause | -|---|---| -| Unexpected keyword argument | yaml argument name differs from adapter parameter name | -| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | -| Unauthorized service | The first matching policy rule excludes that service | -| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | -| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | -| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | -| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | -| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | - -## Deployment platform endpoint - -| Symptom | Likely cause | -|---|---| -| 404 while workflow container is healthy | Workflow function is not named `main` | -| 400 before host receives request | Body is not the platform's `{query: string}` shape | -| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | - -## Cleanup - -| Symptom | Likely cause | -|---|---| -| `ventis clean` succeeds but containers remain | The command removes generated directories only | -| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | -| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py b/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py deleted file mode 100755 index 04baf37..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py +++ /dev/null @@ -1,1257 +0,0 @@ -#!/usr/bin/env python3 -"""Preflight the runtime traps that `ventis build` cannot see. - -This deliberately does not duplicate build-time validation such as malformed -YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those -checks. This script parses Python without importing it and catches failures that -otherwise stay hidden until a container loads an agent, starts a workflow, or -serves its first request. A replica is not evidence: the controller writes -`healthy` to Redis before `_load_agent` runs. - - python validate.py [project_dir] [-c config/global_controller.yaml] - [--json] [--strict] - -Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. - -Runtime capabilities vary across CanyonOS Core installations. This script probes -the importable `ventis` package directly. A capability-gated check reports -UNAVAILABLE when its behavior cannot be proven. -""" - -import argparse -import ast -import builtins -import json -import os -import re -import sys -from typing import ClassVar - -try: - import yaml -except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency - sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") - raise SystemExit(2) from None - - -DEFAULT_CONFIG_PATH = "config/global_controller.yaml" - -# Copied flat into every image over the swept project tree, so a project module -# landing flat under one of these names is overwritten. -# ventis/stub_generator.py generate_docker / generate_workflow_docker. -RUNTIME_FLAT_NAMES = frozenset( - { - "future.py", - "ventis_context.py", - "local_controller.py", - "local_controller_frontend.py", - "redis_client.py", - "grpc_options.py", - "gpu_metrics.py", - "bedrock.py", - "deploy.py", - "session_logging.py", - "workflow_launcher.py", - } -) - -# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. -BASE_AGENT_REQUIREMENTS = [ - "grpcio", - "grpcio-tools", - "redis", - "pyyaml", - "psutil", - "ipdb", - "ipython", - "boto3", -] -# Import name -> distribution name, for the handful where they differ and the -# mismatch would otherwise be reported as a missing requirement. -IMPORT_TO_DISTRIBUTION = { - "attr": "attrs", - "bs4": "beautifulsoup4", - "cv2": "opencv-python", - "dateutil": "python-dateutil", - "dotenv": "python-dotenv", - "grpc": "grpcio", - "grpc_tools": "grpcio-tools", - "jwt": "pyjwt", - "PIL": "pillow", - "psycopg": "psycopg", - "psycopg2": "psycopg2-binary", - "pydantic_settings": "pydantic-settings", - "sklearn": "scikit-learn", - "typing_extensions": "typing-extensions", - "yaml": "pyyaml", -} - -SECRET_PATTERNS = [ - (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), - (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), - (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), - (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), -] -SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) - -ERROR = "ERROR" -WARN = "WARN" -INFO = "INFO" - - -# ------------------------------------------------------------------ # -# Capabilities # -# ------------------------------------------------------------------ # -# -# Stable labels for behavior detected from the importable runtime. They contain -# no external development metadata. - -CAPABILITY_SOURCE = { - "env_file": "runtime env-file injection", - "editable_install": "editable project installation", - "sweeps_all_files": "full project-file sweep", - "stub_two_destinations": "flat and package stub destinations", -} - - -def probe_capabilities(): - """Ask the importable ventis package what it actually supports.""" - caps = dict.fromkeys(CAPABILITY_SOURCE, False) - caps["ventis"] = False - try: - from ventis import stub_generator - except Exception: # noqa: BLE001 - a broken install must not crash the check - return caps - - caps["ventis"] = True - caps["editable_install"] = hasattr(stub_generator, "_install_step") - caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") - caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") - - import importlib - - for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): - try: - module = importlib.import_module(module_name) - except Exception: # noqa: BLE001,S112 - the other path is the live one - continue - if hasattr(module, "resolve_env_file"): - caps["env_file"] = True - break - return caps - - -# ------------------------------------------------------------------ # -# YAML with line numbers # -# ------------------------------------------------------------------ # - - -class LineDict(dict): - """A mapping that remembers where it and each of its keys were written.""" - - line = 0 - key_lines: ClassVar[dict] = {} - - -class LineLoader(yaml.SafeLoader): - pass - - -def _construct_mapping(loader, node): - data = LineDict() - yield data - data.update(loader.construct_mapping(node, deep=False)) - data.line = node.start_mark.line + 1 - data.key_lines = { - key.value: key.start_mark.line + 1 - for key, _ in node.value - if isinstance(key, yaml.ScalarNode) - } - - -LineLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping -) - - -def line_of(mapping, key=None): - """The source line of `key` inside `mapping`, or of the mapping itself.""" - if not isinstance(mapping, LineDict): - return 0 - if key is not None: - return mapping.key_lines.get(key, mapping.line) - return mapping.line - - -def load_yaml(path): - """Parse `path`, returning (data, error). Never raises.""" - try: - with open(path, "r", encoding="utf-8") as handle: - return yaml.load(handle, Loader=LineLoader), None - except Exception as exc: # noqa: BLE001 - any parse failure is a finding - return None, str(exc) - - -# ------------------------------------------------------------------ # -# Findings # -# ------------------------------------------------------------------ # - - -class Report: - def __init__(self, project_dir, capabilities): - self.project_dir = project_dir - self.capabilities = capabilities - self.findings = [] - # A peer agent is imported by the name of its generated stub, which the - # build copies flat into every image. Those are not project modules and - # need no requirement. - self.stub_module_names = set() - - def add(self, check, level, path, line, summary, mechanism): - self.findings.append( - { - "check": check, - "level": level, - "path": self.rel(path) if path else "", - "line": line or 0, - "summary": summary, - "mechanism": mechanism, - } - ) - - def error(self, check, path, line, summary, mechanism): - self.add(check, ERROR, path, line, summary, mechanism) - - def warn(self, check, path, line, summary, mechanism): - self.add(check, WARN, path, line, summary, mechanism) - - def unavailable(self, check, summary): - self.add(check, INFO, "", 0, summary, "") - - def rel(self, path): - try: - return os.path.relpath(path, self.project_dir) - except ValueError: - return path - - def counts(self): - errors = sum(1 for f in self.findings if f["level"] == ERROR) - warnings = sum(1 for f in self.findings if f["level"] == WARN) - return errors, warnings - - -# ------------------------------------------------------------------ # -# Python source helpers # -# ------------------------------------------------------------------ # - - -def parse_python(path): - """AST for `path`, or (None, error). The port is never imported.""" - try: - with open(path, "r", encoding="utf-8") as handle: - source = handle.read() - except OSError as exc: - return None, str(exc) - try: - return ast.parse(source, filename=path), None - except SyntaxError as exc: - return None, f"{exc.msg} (line {exc.lineno})" - - -def find_class(tree, name): - for node in tree.body: - if isinstance(node, ast.ClassDef) and node.name == name: - return node - return None - - -def class_methods(class_node): - return { - node.name: node - for node in class_node.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - } - - -def parameter_names(func_node): - """Every parameter a caller can pass by keyword, minus self.""" - args = func_node.args - positional = [a.arg for a in args.posonlyargs + args.args] - if positional and positional[0] in ("self", "cls"): - positional = positional[1:] - return positional + [a.arg for a in args.kwonlyargs] - - -def required_parameters(func_node): - """Parameters with no default, minus self.""" - args = func_node.args - positional = [a.arg for a in args.posonlyargs + args.args] - if positional and positional[0] in ("self", "cls"): - positional = positional[1:] - if args.defaults: - positional = positional[: len(positional) - len(args.defaults)] - kwonly = [ - arg.arg - for arg, default in zip(args.kwonlyargs, args.kw_defaults) - if default is None - ] - return positional + kwonly - - -def toplevel_import_names(tree): - """Top-level package name of every import in the module, with line numbers.""" - names = {} - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - names.setdefault(alias.name.split(".")[0], node.lineno) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - names.setdefault(node.module.split(".")[0], node.lineno) - return names - - -# ------------------------------------------------------------------ # -# V006-V010 adapter failures hidden by _load_agent # -# ------------------------------------------------------------------ # - -BUILTIN_TYPE_NAMES = frozenset( - name - for name in dir(builtins) - if isinstance(getattr(builtins, name), type) and not name[0].isupper() -) - - -def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): - """V006 V007 V008 V009 V010.""" - name = agent_block["name"] - functions = agent_block.get("functions") or [] - check_argument_types(report, agent_yaml_path, functions) - - entrypoint = entry.get("entrypoint") - if not entrypoint: - return - entrypoint_path = os.path.join(project_dir, entrypoint) - if not os.path.isfile(entrypoint_path): - return - - tree, error = parse_python(entrypoint_path) - if tree is None: - report.error( - "V006", - entrypoint_path, - 0, - f"the entrypoint does not parse: {error}", - "_load_agent exec_module's it and swallows the exception; the first " - "request answers 'No agent loaded'.", - ) - return - - class_node = find_class(tree, name) - if class_node is None: - classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] - found = ", ".join(classes) if classes else "no classes at all" - report.error( - "V006", - entrypoint_path, - 1, - f"no class named `{name}` at module level (found: {found})", - "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " - "the AttributeError. The class name must equal agent.name exactly.", - ) - return - - methods = class_methods(class_node) - check_constructor(report, entrypoint_path, name, methods) - - for func in functions: - if not isinstance(func, dict) or not isinstance(func.get("name"), str): - continue - check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) - - -def check_argument_types(report, agent_yaml_path, functions): - """V010 -- the type string is pasted into an ast.Name, never checked.""" - for func in functions: - if not isinstance(func, dict): - continue - for arg in func.get("arguments") or []: - if not isinstance(arg, dict) or "type" not in arg: - continue - declared = arg.get("type") - if not isinstance(declared, str): - continue # stub generation reports malformed type values - if declared in BUILTIN_TYPE_NAMES: - continue - report.error( - "V010", - agent_yaml_path, - line_of(arg, "type"), - f"`type: {declared}` is not a builtin", - "stub_generator pastes it verbatim into the generated " - "annotation, and the stub module imports only Future and " - "inspect. Anything else raises NameError when the stub is " - "imported -- after a green build. Use str int float bool dict " - "list.", - ) - - -def check_constructor(report, entrypoint_path, name, methods): - """V007 -- _load_agent calls agent_class() with no arguments.""" - init = methods.get("__init__") - if init is None: - return - required = required_parameters(init) - if required: - report.error( - "V007", - entrypoint_path, - init.lineno, - f"`{name}.__init__` requires {', '.join(required)}", - "_load_agent calls agent_class() with no arguments; the TypeError is " - "swallowed and the first request answers 'No agent loaded'. Read " - "configuration from the environment inside __init__ instead.", - ) - - -def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): - """V008 V009.""" - func_name = func["name"] - method = methods.get(func_name) - if method is None: - report.error( - "V008", - entrypoint_path, - 0, - f"`{class_name}` has no method `{func_name}`", - "The yaml declares it, so callers get a stub for it; the controller " - f"then answers \"Agent {class_name} has no method '{func_name}'\".", - ) - return - - # V009 -- nothing on the execution path awaits. - if isinstance(method, ast.AsyncFunctionDef): - report.error( - "V009", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` is `async def`", - "The executor calls method(**args) with no await, so Redis receives " - "''. Keep the signature synchronous and call " - "asyncio.run(...) inside the body.", - ) - - # V008 -- the controller calls method(**args) with the yaml's names. - declared = [ - arg["name"] - for arg in func.get("arguments") or [] - if isinstance(arg, dict) and isinstance(arg.get("name"), str) - ] - actual = parameter_names(method) - required = required_parameters(method) - - missing = [arg for arg in declared if arg not in actual] - if missing: - report.error( - "V008", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` has no parameter " - f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", - "LocalController does method(**args) with the yaml's argument names. " - "A mismatch is TypeError: unexpected keyword argument, at request " - f"time. See {os.path.basename(agent_yaml_path)}.", - ) - - unfilled = [arg for arg in required if arg not in declared] - if unfilled: - report.error( - "V008", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " - "the yaml does not declare", - "Only declared arguments are ever sent, and the generated stub gives " - "none of them a default. Declare them in the yaml or default them " - "in the signature.", - ) - - - -# ------------------------------------------------------------------ # -# V016-V018 the workflow # -# ------------------------------------------------------------------ # - - -def check_stub_imports(report, workflow_path, tree, stub_classes): - """V023 -- the workflow must import a stub as `from agents. import `. - - The build copies each stub to exactly one path, and for the workflow image - that path is agents/.py. Two ways of writing this line fail, and - the project walks you into both: the flat form is what examples/ uses, and - the class name is the one `ventis build` prints, which is not the one it - writes. - """ - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - base = alias.name.split(".")[0] - if base in stub_classes: - report.error( - "V023", workflow_path, node.lineno, - f"`import {alias.name}` -- the stub is at " - f"agents/{base}.py, not flat", - "The build copies a stub to one path, and for the " - "workflow that path is under agents/. This is a " - "ModuleNotFoundError the moment the workflow runs. " - f"Write `from agents.{base} import {stub_classes[base]}`.", - ) - continue - - if not isinstance(node, ast.ImportFrom) or not node.module: - continue - - module = node.module - if module in stub_classes: - report.error( - "V023", workflow_path, node.lineno, - f"`from {module} import ...` -- the stub is at " - f"agents/{module}.py, not flat", - "The build copies a stub to one path, and for the workflow " - "that path is under agents/. The flat form is what this " - "repository's own examples use and it raises " - "ModuleNotFoundError in the workflow image. Write " - f"`from agents.{module} import {stub_classes[module]}`.", - ) - continue - - if not module.startswith("agents."): - continue - base = module.split(".", 1)[1] - expected = stub_classes.get(base) - if expected is None: - continue - for alias in node.names: - if alias.name == expected: - continue - if alias.name == f"{expected}Stub": - report.error( - "V023", workflow_path, node.lineno, - f"`{alias.name}` is the name the build prints, not the " - f"class it writes", - "generate_agent_stub sets class_name = agent_config['name'] " - "and then recomputes it with a 'Stub' suffix for the log " - "line only. The message names a class that does not exist; " - f"the class is `{expected}`.", - ) - else: - report.error( - "V023", workflow_path, node.lineno, - f"`{alias.name}` is not a class the stub for {base} defines", - f"The stub's class carries the agent's own name: `{expected}`.", - ) - - -def check_workflow(report, workflow_path, stub_classes=None): - """V016 V017 V018 V023.""" - tree, error = parse_python(workflow_path) - if tree is None: - report.error("V016", workflow_path, 0, f"does not parse: {error}", "") - return - - main = None - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( - node.name == "main" - ): - main = node - break - - if main is None: - defined = [ - n.name - for n in tree.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) - ] - found = ", ".join(defined) if defined else "no top-level functions" - report.error( - "V016", - workflow_path, - 1, - f"no top-level function named `main` (found: {found})", - "CanyonOS Core serves POST /, but the deployment platform's " - "test endpoint posts to a hardcoded /main. A differently named " - "workflow builds, deploys and stays unreachable -- 404, container " - "healthy.", - ) - else: - check_main_signature(report, workflow_path, main) - - if stub_classes: - check_stub_imports(report, workflow_path, tree, stub_classes) - - # V016 -- deploy() is what starts Flask. - if not any( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "deploy" - for node in ast.walk(tree) - ): - report.error( - "V016", - workflow_path, - 1, - "the workflow never calls `deploy(...)`", - "workflow_launcher.py exec's this file and nothing else starts the " - "HTTP server; the container comes up serving nothing.", - ) - - check_main_guard(report, workflow_path, tree) - check_fused_fanout(report, workflow_path, tree) - - -def check_main_signature(report, workflow_path, main): - """V016 -- the platform sends exactly {"query": ...}.""" - if isinstance(main, ast.AsyncFunctionDef): - report.error( - "V016", - workflow_path, - main.lineno, - "`main` is `async def`", - "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " - "no await; the response body would be a coroutine repr.", - ) - params = parameter_names(main) - if not params: - report.error( - "V016", - workflow_path, - main.lineno, - "`main` takes no arguments", - 'The platform posts {"query": "..."} and deploy() splats the ' - "body in as kwargs -- TypeError on every request.", - ) - return - if params[0] != "query": - report.error( - "V016", - workflow_path, - main.lineno, - f"`main`'s first parameter is `{params[0]}`, not `query`", - "The platform's body schema is strictly validated as " - "{query: string}; any other key is rejected with 400 in the control " - "plane, before the request reaches the host.", - ) - extra = [p for p in required_parameters(main) if p != "query"] - if extra: - report.error( - "V016", - workflow_path, - main.lineno, - f"`main` requires {', '.join(extra)} beyond `query`", - "Only `query` is ever sent, so every other parameter needs a " - "default or the call raises on every request. Pack richer input " - "into `query`.", - ) - - -def check_main_guard(report, workflow_path, tree): - """V017 -- the workflow is exec'd, so __name__ == "__main__".""" - for node in tree.body: - if not isinstance(node, ast.If): - continue - test = node.test - if ( - isinstance(test, ast.Compare) - and isinstance(test.left, ast.Name) - and test.left.id == "__name__" - and any( - isinstance(c, ast.Constant) and c.value == "__main__" - for c in test.comparators - ) - ): - report.error( - "V017", - workflow_path, - node.lineno, - '`if __name__ == "__main__":` block in the workflow', - "workflow_launcher.py runs exec(open().read()), so " - "__name__ IS '__main__' here and this block executes in " - "production, at container start.", - ) - - -def check_fused_fanout(report, workflow_path, tree): - """V018 -- .value() blocks, so dispatching and resolving in one - comprehension runs the fan-out one call at a time.""" - comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) - for node in ast.walk(tree): - if not isinstance(node, comprehensions + (ast.DictComp,)): - continue - elements = ( - [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] - ) - for element in elements: - for inner in ast.walk(element): - if ( - isinstance(inner, ast.Call) - and isinstance(inner.func, ast.Attribute) - and inner.func.attr == "value" - and isinstance(inner.func.value, ast.Call) - ): - report.error( - "V018", - workflow_path, - node.lineno, - "one comprehension both dispatches a call and resolves " - "it with .value()", - ".value() blocks, so each call completes before the " - "next is dispatched. It does not error -- the fan-out " - "is just silently serial, and with it the reason to be " - "on CanyonOS Core. Dispatch every call first, then resolve: " - "futures = [a.work(i) for i in items] then " - "[f.value() for f in futures].", - ) - return - - -# ------------------------------------------------------------------ # -# V019-V020 what the copy order overwrites # -# ------------------------------------------------------------------ # - - -def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): - """V019 V020 -- later copies land on earlier ones at the context root.""" - for entry in sorted(os.listdir(project_dir)): - path = os.path.join(project_dir, entry) - if not os.path.isfile(path) or not entry.endswith(".py"): - continue - - # V019 -- the shared runtime is copied flat, after the project sweep. - if entry in RUNTIME_FLAT_NAMES: - report.error( - "V019", - path, - 1, - f"a project module named `{entry}` sits at the project root", - "The shared CanyonOS Core runtime is copied flat into the image after " - "the project sweep, so this file is overwritten by CanyonOS Core's own " - f"{entry}. Rename it or move it into a package directory.", - ) - - # V020 -- a stub is copied flat under its yaml's basename. - entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} - for yaml_path in yaml_paths: - stem = os.path.splitext(os.path.basename(yaml_path))[0] - module = f"{stem}.py" - if module in entrypoint_basenames: - continue # the entrypoint is copied last and wins its flat name back - candidate = os.path.join(project_dir, module) - if os.path.isfile(candidate): - report.error( - "V020", - candidate, - 1, - f"`{os.path.basename(yaml_path)}` generates a stub that lands on " - f"`{module}`", - "The yaml's basename names the stub, and the stub is copied flat " - "over the swept tree. Anything importing this module inside the " - "container gets the generated stub instead of the real code. " - "Rename the yaml to match its own entrypoint.", - ) - - -# ------------------------------------------------------------------ # -# V030-V031 capability-gated rules # -# ------------------------------------------------------------------ # - - -def check_env_file(report, config, config_path, project_dir): - """V030 -- gated on detected env-file injection support.""" - declared = config.get("env_file") - supported = report.capabilities.get("env_file") - - if not supported: - if declared: - report.error( - "V030", - config_path, - line_of(config, "env_file"), - f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", - "No resolve_env_file in the importable ventis package, so the " - "key is silently dropped and the container answers a provider " - "credential error on the first request. This port requires the " - "`env_file` runtime capability.", - ) - else: - report.unavailable( - "V030", - "env_file is not supported by the importable `ventis` runtime. " - "Credentials have no declared path into a container on this tree.", - ) - return - - if not declared: - report.warn( - "V030", - config_path, - line_of(config), - "no `env_file:` in the config", - "Only runtime-managed VENTIS_* variables are guaranteed without it. " - "If the source reads credentials from the environment, the first " - "request fails on a provider error.", - ) - return - - # Path existence and readability are deploy-preflight checks. Do not - # duplicate them here. - - -def check_import_root(report, project_dir, entrypoint_paths): - """V031 -- gated on detected editable-install support.""" - supported = report.capabilities.get("editable_install") - has_metadata = any( - os.path.isfile(os.path.join(project_dir, name)) - for name in ("pyproject.toml", "setup.py", "setup.cfg") - ) - - non_flat = [] - for path in entrypoint_paths: - tree, _ = parse_python(path) - if tree is None: - continue - for name, lineno in toplevel_import_names(tree).items(): - if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: - continue - if _resolves_flat(project_dir, name): - continue - location = _resolves_nested(project_dir, name) - if location: - non_flat.append((path, lineno, name, location)) - - if not supported: - report.unavailable( - "V031", - "the editable install (`-e .`) is not supported by the importable " - "`ventis` runtime. Only names rooted at /app import inside a container.", - ) - for path, lineno, name, location in non_flat: - report.error( - "V031", - path, - lineno, - f"`import {name}` resolves to {location}, which is not at the " - "project root", - "sys.path[0] is /app and this CanyonOS Core runs no editable install, " - "so only modules swept to the root import. The adapter raises " - "ModuleNotFoundError inside _load_agent and the first request " - "answers 'No agent loaded'.", - ) - return - - if non_flat and not has_metadata: - for path, lineno, name, location in non_flat: - report.error( - "V031", - path, - lineno, - f"`import {name}` resolves to {location}, and the project root " - "has no packaging metadata", - "A pyproject.toml, setup.py or setup.cfg at the port root is " - "what adds `-e .`; metadata nested inside the untouched source " - "tree is ignored. Add minimal root metadata pointing at the " - "existing package directory. Without it the install is skipped " - "silently.", - ) - - -def _resolves_flat(project_dir, name): - """Whether Python can resolve `name` with /app as its import root. - - A directory does not need __init__.py: PEP 420 namespace packages resolve - from sys.path just like regular packages. - """ - return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( - os.path.join(project_dir, name) - ) - - -def _resolves_nested(project_dir, name): - """Where below /app `name` lives but cannot resolve as a top-level name.""" - for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] - if root == project_dir: - continue - if f"{name}.py" in files: - return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) - if name in dirs: - return os.path.relpath(os.path.join(root, name), project_dir) - return None - - -# ------------------------------------------------------------------ # -# W003, W006 secrets and imports a green build does not reject # -# ------------------------------------------------------------------ # - - -def check_secrets(report, port_paths): - """W003 -- env_file is the way in; nothing else is.""" - for path in port_paths: - try: - with open(path, "r", encoding="utf-8") as handle: - lines = handle.read().splitlines() - except OSError: - continue - for number, line in enumerate(lines, start=1): - for pattern, description in SECRET_PATTERNS: - if pattern.search(line): - report.warn( - "W003", - path, - number, - f"this line looks like {description}", - "Never put a secret in the source tree or the build " - "context. The build sweeps the project into every " - "image.", - ) - break - - tree, _ = parse_python(path) - if tree is None: - continue - for node in ast.walk(tree): - if not isinstance(node, ast.Assign): - continue - if not ( - isinstance(node.value, ast.Constant) - and isinstance(node.value.value, str) - and node.value.value.strip() - ): - continue - for target in node.targets: - if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): - report.warn( - "W003", - path, - node.lineno, - f"`{target.id}` is assigned a literal string", - "Read it from the environment instead; the build sweeps " - "this file into every image.", - ) - - -def _pyproject_dependencies(project_dir): - """What `-e .` installs alongside `requirements:`, or None if unreadable. - - None and the empty set mean different things here: empty means the project - declares no dependencies, None means we could not find out -- a setup.py, or - a tomllib this interpreter does not have. The caller must not treat the - second as the first, or it warns about imports the install would satisfy. - """ - path = os.path.join(project_dir, "pyproject.toml") - if not os.path.isfile(path): - return None - try: - import tomllib - except ImportError: # < 3.11 - return None - try: - with open(path, "rb") as handle: - data = tomllib.load(handle) - except Exception: # noqa: BLE001 - malformed metadata is uv's error to give - return None - deps = (data.get("project") or {}).get("dependencies") - if not isinstance(deps, list): - return None - return {_normalize_distribution(d) for d in deps if isinstance(d, str)} - - -def check_requirements_coverage( - report, project_dir, entry, entrypoint_path, config_path -): - """W006 -- an import the container cannot satisfy.""" - tree, _ = parse_python(entrypoint_path) - if tree is None: - return - - declared = { - _normalize_distribution(item) - for item in (entry.get("requirements") or []) - if isinstance(item, str) - } - - # Where the editable install exists, `-e .` resolves the project's own - # [project.dependencies] in the same pass as `requirements:`. Warning about - # those is a false positive, and a false warning about a dependency is worse - # than none: it teaches the reader to dismiss this check. - editable = report.capabilities.get("editable_install") - metadata = any( - os.path.isfile(os.path.join(project_dir, name)) - for name in ("pyproject.toml", "setup.py", "setup.cfg") - ) - unreadable_metadata = False - if editable and metadata: - project_deps = _pyproject_dependencies(project_dir) - if project_deps is None: - unreadable_metadata = True - else: - declared |= project_deps - base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} - stdlib = getattr(sys, "stdlib_module_names", frozenset()) - - for name, lineno in sorted(toplevel_import_names(tree).items()): - if name in stdlib or name == "ventis": - continue - # Provided by the image itself: the shared runtime is copied flat, and - # every agents/*.yaml generates a stub that is copied flat too. - if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: - continue - if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): - continue - distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) - if distribution in base or distribution in declared: - continue - if unreadable_metadata: - mechanism = ( - "The container installs the base list, `requirements:`, and -- " - "since this project declares packaging metadata -- whatever " - "`-e .` resolves from it. That metadata could not be read here, " - f"so if it already requires `{name}` this line is noise; " - "otherwise it is a ModuleNotFoundError inside _load_agent and " - "'No agent loaded' on the first request." - ) - else: - mechanism = ( - "The container installs the base list plus `requirements:` and " - "nothing else, so this is a ModuleNotFoundError inside " - "_load_agent and 'No agent loaded' on the first request. If the " - f"distribution is named something other than `{name}`, declare " - f"that name in {report.rel(config_path)}." - ) - report.warn( - "W006", - entrypoint_path, - lineno, - f"`import {name}` is in neither the runtime's base list nor this " - "entry's `requirements:`", - mechanism, - ) - - -def _normalize_distribution(name): - return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( - "_", "-" - ) - - -# ------------------------------------------------------------------ # -# Driver # -# ------------------------------------------------------------------ # - - -def validate(project_dir, config_path, capabilities): - """Inspect only failures hidden behind a successful image build.""" - report = Report(project_dir, capabilities) - - # The build owns config/YAML syntax and shape validation. We read only enough - # valid structure to locate code for the deeper checks below. - config, error = load_yaml(config_path) - if error is not None or not isinstance(config, dict): - report.unavailable( - "BUILD", - "runtime preflight skipped because the config cannot be read; " - "ventis build owns and reports this error.", - ) - return report - - import glob - - yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) - report.stub_module_names = { - os.path.splitext(os.path.basename(path))[0] for path in yaml_paths - } - - agents_by_name = {} - stub_classes = {} - for path in yaml_paths: - data, yaml_error = load_yaml(path) - agent = data.get("agent") if isinstance(data, dict) else None - name = agent.get("name") if isinstance(agent, dict) else None - if yaml_error is not None or not isinstance(name, str): - continue # ventis build reports malformed agent declarations - agents_by_name[name] = (path, agent) - stub_classes[os.path.splitext(os.path.basename(path))[0]] = name - - entries = config.get("agents") - if not isinstance(entries, list): - report.unavailable( - "BUILD", - "runtime preflight skipped because `agents:` is not a list; " - "ventis build owns and reports this error.", - ) - return report - - entrypoints = [] - for entry in entries: - if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": - continue - name = entry.get("name") - entrypoint = entry.get("entrypoint") - if isinstance(entrypoint, str): - entrypoints.append(entrypoint) - if name in agents_by_name: - yaml_path, agent_block = agents_by_name[name] - check_adapter(report, yaml_path, agent_block, entry, project_dir) - entrypoint_path = os.path.join(project_dir, entrypoint or "") - if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): - check_requirements_coverage( - report, project_dir, entry, entrypoint_path, config_path - ) - - for entry in entries: - if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": - continue - workflow_file = entry.get("workflow_file") - if not isinstance(workflow_file, str): - continue - workflow_path = os.path.join(project_dir, workflow_file) - if os.path.isfile(workflow_path): - check_workflow(report, workflow_path, stub_classes) - - # These survive a green build and otherwise surface only in a container or - # on its first request. - check_flat_collisions(report, project_dir, yaml_paths, entrypoints) - check_env_file(report, config, config_path, project_dir) - - entrypoint_paths = [ - os.path.join(project_dir, e) - for e in entrypoints - if os.path.isfile(os.path.join(project_dir, e)) - ] - check_import_root(report, project_dir, entrypoint_paths) - - port_paths = list(entrypoint_paths) - for entry in entries: - if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): - candidate = os.path.join(project_dir, entry["workflow_file"]) - if os.path.isfile(candidate): - port_paths.append(candidate) - - # Secret detection remains because a green image build would permanently - # bake the credential into every image. - check_secrets(report, port_paths) - return report - - - -# ------------------------------------------------------------------ # -# Output # -# ------------------------------------------------------------------ # - -LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} - - -def _wrap(text, width, indent): - words = text.split() - lines = [] - current = "" - for word in words: - candidate = f"{current} {word}".strip() - if len(candidate) + len(indent) > width and current: - lines.append(indent + current) - current = word - else: - current = candidate - if current: - lines.append(indent + current) - return lines - - -def print_report(report, project_dir): - caps = report.capabilities - if not caps.get("ventis"): - print("ventis is not importable here -- capability-gated rules are") - print("reported UNAVAILABLE rather than checked.\n") - else: - print("CanyonOS Core capabilities detected:") - for key, source in CAPABILITY_SOURCE.items(): - mark = "yes" if caps.get(key) else "no " - print(f" {mark} {key:<22} {source}") - print() - - findings = sorted( - report.findings, - key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), - ) - for finding in findings: - where = finding["path"] - if where and finding["line"]: - where = f"{where}:{finding['line']}" - header = f"{finding['check']} {finding['level']:<5}" - print(f"{header} {where}" if where else header) - for line in _wrap(finding["summary"], 78, " "): - print(line) - if finding["mechanism"]: - for line in _wrap(finding["mechanism"], 78, " "): - print(line) - print() - - errors, warnings = report.counts() - if not findings: - print(f"{project_dir}: clean.") - return - print(f"{errors} error(s), {warnings} warning(s).") - - -def main(argv=None): - parser = argparse.ArgumentParser( - description="Check a CanyonOS Core port against the rules in SKILL.md." - ) - parser.add_argument( - "project_dir", nargs="?", default=".", help="the port's project root" - ) - parser.add_argument( - "-c", - "--config", - default=DEFAULT_CONFIG_PATH, - help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", - ) - parser.add_argument("--json", action="store_true", help="emit findings as JSON") - parser.add_argument( - "--strict", action="store_true", help="fail on warnings as well as errors" - ) - args = parser.parse_args(argv) - - project_dir = os.path.abspath(args.project_dir) - config_path = ( - args.config - if os.path.isabs(args.config) - else os.path.join(project_dir, args.config) - ) - - capabilities = probe_capabilities() - report = validate(project_dir, config_path, capabilities) - errors, warnings = report.counts() - - if args.json: - print( - json.dumps( - { - "project_dir": project_dir, - "capabilities": capabilities, - "errors": errors, - "warnings": warnings, - "findings": report.findings, - }, - indent=2, - ) - ) - else: - print_report(report, report.rel(project_dir) or project_dir) - - if errors or (args.strict and warnings): - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From c12c5c5b023c5c7c4a38d5273feac98b5665aa81 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 12:31:18 -0700 Subject: [PATCH 27/31] removed more useless code --- .../skills/porting-to-canyonos-core/SKILL.md | 240 ---- .../references/ec2.md | 41 - .../references/llm-proxy.md | 64 - .../references/packaging.md | 81 -- .../references/runtime-contract.md | 187 --- .../references/troubleshooting.md | 60 - .../porting-to-canyonos-core/validate.py | 1257 ----------------- 7 files changed, 1930 deletions(-) delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/SKILL.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/ec2.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/packaging.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md delete mode 100755 cli/.claude/skills/porting-to-canyonos-core/validate.py diff --git a/cli/.claude/skills/porting-to-canyonos-core/SKILL.md b/cli/.claude/skills/porting-to-canyonos-core/SKILL.md deleted file mode 100644 index fe6dd49..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/SKILL.md +++ /dev/null @@ -1,240 +0,0 @@ ---- -name: porting-to-canyonos-core -description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. -compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. ---- - -# Port an agent project to CanyonOS Core - -CanyonOS Core is the product name. Its compatibility executable and Python -package remain `ventis`; environment variables and Docker resources retain the -`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding -strings. Do not rename them. - -## Load references only when needed - -- Read [references/packaging.md](references/packaging.md) when a source import - does not resolve from `/app`, the source is nested, or packaging metadata is - involved. -- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target - includes `llm_proxy`. -- Read [references/ec2.md](references/ec2.md) only when any config entry uses - `provider: EC2`. -- Read [references/troubleshooting.md](references/troubleshooting.md) after a - failed build, image probe, deploy, or request. -- Read [references/runtime-contract.md](references/runtime-contract.md) when a - validator finding needs explanation or the runtime mechanism is unclear. - -## Goal: thin scaffolding beside untouched source - -```text -agents/.yaml one callable surface per service -agents/.py one thin adapter per service, when needed -workflow/_workflow.py HTTP entry point; calls deploy() -config/global_controller.yaml deployment manifest -config/policy.yaml optional access restriction -pyproject.toml conditional nested-import scaffolding - unchanged -``` - -The file count follows the deployment. A multi-agent port has one yaml/adapter -pair per service that is worth deploying separately. If a source class already -satisfies the runtime contract, point its config entry at that file and do not -copy it into an adapter. - -Everything the source already owns—prompts, tools, schemas, parsing, retries, -model clients, and node bodies—is imported. The port re-expresses only the -CanyonOS Core boundary and framework-owned orchestration. - -The port root is the existing repository root and the directory from which -`ventis build` runs. Write scaffolding there beside existing directories. If the -repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, -and `config/` beside it. Never move or copy the repository into a new `src/` -directory, and never create an outer wrapper merely for the port. - -## 1. Survey before writing - -Identify: - -1. The source entry point and callable input/output. -2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, - `Send`, `Command`, interrupts). -3. Runtime-injected services nodes read: stores, context, memory, sessions, or - callback managers. -4. Sync versus async boundaries. -5. Imports and declared runtime dependencies. -6. Model provider, credential names, streaming use, and optional `llm_proxy`. -7. Whether independent work fans out and benefits from separate replicas. -8. Whether source imports resolve from the project root that becomes `/app`. - -Run the validator once now. Its header detects capabilities directly from the -importable runtime rather than external development metadata: - -```bash -python /validate.py . -``` - -If config or agent yaml is malformed, the validator defers to `ventis build`. -Capability-gated findings say which runtime behavior is available. - -## 2. Choose service boundaries - -Start with one service. Split only when it creates independent parallel work or -a distinct resource/replica profile. - -- Keep a ReAct loop together; every turn needs shared message history. -- Hoist supervisor task lists and `Send`-style fan-out into the workflow. -- Do not create a one-replica service with no distinct resource profile merely - to mirror every source graph node. - -Rewrite framework-owned edges as ordinary Python. Import the connected node -functions unchanged. Construct runtime-injected service objects from source -configuration; do not invent models, dimensions, stores, or defaults silently. -Report any choice the source does not specify. - -## 3. Write declarations and adapters - -### Agent yaml - -Use one yaml per deployed service. Argument types are bare builtins only: -`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is -required by the generated stub. `returns.type` is documentation; use `dict` or -`list` to signal that workflow callers must `json.loads` the returned string. - -### Adapter - -The entrypoint exposes a module-level class named exactly `agent.name`. It -constructs with no arguments and its declared methods are synchronous. Read -configuration from the environment in `__init__`. Bridge source coroutines -inside a synchronous method with `asyncio.run(...)`. Serialize framework objects -with their own JSON-safe serializer before returning. - -Do not duplicate source prompts, tools, schemas, or model calls. Keep the source -provider and SDK. - -### Workflow - -Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. -Import generated stubs by yaml basename and agent class name: - -```python -from deploy import deploy -from agents. import -``` - -The deployment platform sends `{query: string}` to `/main`. Pack richer input -inside `query`; any additional workflow parameter has a default. - -Dispatch every remote call before resolving any future: - -```python -futures = [agent.work(item=item) for item in items] -results = [json.loads(future.value()) for future in futures] -``` - -Do not fuse dispatch and `.value()` in one comprehension; that silently -serializes fan-out. Do not add an `if __name__ == "__main__":` block: the -workflow is executed with `__name__ == "__main__"` in production. - -### Config - -For each service, keep these names aligned: - -```text -config entry name == yaml agent.name == entrypoint class name -``` - -Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a -list of distribution-name strings. Put `env_file` at config top level when the -runtime capability is available. Omit `policy.yaml` unless access must be -restricted; if present, give it a non-empty `rules` list. - -## Hard rules - -Capitalized **MUST** and **NEVER** are reserved for port-breaking or -source-integrity rules. The owner column states where each is decided. - -| ID | Rule | Owner | -|---|---|---| -| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | -| M2 | The class MUST construct with no arguments | V007 | -| M3 | yaml argument names MUST match Python parameter names | V008 | -| M4 | yaml argument types MUST be bare builtins | V010 | -| M5 | Declared adapter methods MUST be synchronous | V009 | -| M6 | Config names MUST match yaml agent names | build | -| M7 | Config names MUST not collide after lowercase normalization | build output | -| M8 | Local provider MUST be lowercase `local` | deploy preflight | -| M9 | `replicas` MUST be an integer | deploy preflight | -| M10 | `requirements` MUST be a list of strings | build | -| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | -| M12 | Workflow MUST NEVER contain a main guard | V017 | -| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | -| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | -| M15 | Workflow MUST import stubs from `agents.` | V023 | -| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | -| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | -| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | -| M19 | NEVER hardcode or bake a real credential into an image | W003 | -| M20 | NEVER edit or vendor the source tree | `git status` | -| M21 | NEVER swap the source LLM provider | review | -| M22 | NEVER silently move, drop, or reclassify source dependencies | review | -| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | -| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | - -## 4. Validate, build, and probe - -Run static preflight, then let the build own build-time validation: - -```bash -python /validate.py . -ventis build -c config/global_controller.yaml -``` - -A green build never imports the adapter. Probe each agent image in this order: - -```bash -# Runtime startup path - docker run --rm ventis- \ - python -c "import local_controller" - -# Agent load path; include --env-file when configured - docker run --rm --env-file ventis- \ - python -c "import importlib.util,sys; \ -s=importlib.util.spec_from_file_location('m','.py'); \ -m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ -m.();print('ok')" -``` - -Also probe the workflow image with `python -c "import local_controller"`; it has -its own dependency resolve and generated-stub imports. - -Then deploy, send a representative request, and poll its status: - -```bash -ventis deploy -c config/global_controller.yaml -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query":""}' -curl http://localhost:8080/status/ -``` - -A successful outer request with a source-level failure still proves the port -reached and returned the source behavior. Record the distinction. - -## 5. Clean up - -After collecting evidence, stop foreground deploy with Ctrl+C and wait for -controller cleanup. Remove exact leftovers if startup crashed. Then remove build -products and exact images from this config: - -```bash -ventis clean -docker image rm ventis- \ - ventis- - -test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container -docker ps -a --format '{{.Names}}' -``` - -`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it -does not remove containers or images. Keep port scaffolding, untouched source, -and requested logs or reports. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md b/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md deleted file mode 100644 index e06daa0..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md +++ /dev/null @@ -1,41 +0,0 @@ -# EC2 deployment - -Read this only when at least one config entry uses `provider: EC2`. - -## Configuration - -Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The -top-level `ec2` block supplies the runtime's required infrastructure and SSH -settings. Read the target checkout's deploy preflight and EC2 runtime before -writing the block; do not copy values from an example environment. - -Typical required categories are: - -- AMI and instance type -- region and subnet -- security groups -- SSH user and credentials accepted by the runtime - -`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof -that provisioning, SSH, image transfer, or remote container startup works. - -## Networking - -A remote container's `host.docker.internal` names its own EC2 Docker host. It -does not name the local controller machine. Databases, model proxies, and other -services must use addresses reachable from every selected host. - -The environment file may be copied temporarily to a remote host by runtimes that -expose the `env_file` capability. Confirm behavior from the capability probe and -target runtime rather than assuming local Docker semantics. - -## Probes and cleanup - -Run the same runtime and adapter probes against the exact image before remote -deployment. After deploy, verify the remote container logs; controller health -can be green even when agent loading failed. - -Stop foreground deploy normally so the controller can terminate recorded EC2 -instances. If provisioning or startup fails before an instance is recorded, -inspect the cloud provider directly and remove exact leaked resources. Never use -a broad cleanup command against unrelated instances. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md deleted file mode 100644 index f726bd7..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md +++ /dev/null @@ -1,64 +0,0 @@ -# LLM proxy integration - -Read this only when the target checkout contains `llm_proxy` or the deployment -explicitly routes model SDKs through it. - -## Preserve provider protocols - -The proxy redirects provider endpoints; it does not convert providers. Keep the -source SDK, model ID, request body, and response parsing unchanged. - -Configure only the provider variables the source uses: - -```dotenv -OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 -ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic -AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock -``` - -Some SDKs refuse to initialize without caller credentials. Give agent containers -non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS -credentials in the separate proxy process, not in the port's `env_file`. - -## Start locally - -The proxy defaults conflict with a typical deployment: host loopback is not -reachable from a container, and port 8080 is normally used by the workflow API. -Use a non-loopback bind and a different port: - -```bash -PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy -curl http://127.0.0.1:8081/healthz -``` - -Local CanyonOS Core containers resolve `host.docker.internal` through their -Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the -machine running `ventis deploy`. Distributed deployments need a reachable proxy -address or one proxy on each host. - -## Supported call shape - -The implementation buffers complete requests and responses: - -- OpenAI and Anthropic non-streaming HTTP calls are forwarded. -- Bedrock `invoke` is reissued through the proxy's boto3 client. -- OpenAI/Anthropic streaming is unsupported. -- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are - unsupported. - -Survey the source before selecting the proxy. Do not silently disable streaming; -report the unsupported behavior and stop. - -## Credential behavior - -- The OpenAI adapter removes caller authorization and inserts the proxy key. -- The Anthropic adapter removes caller key headers and inserts the proxy key. -- Botocore still signs requests sent to a custom endpoint, so a caller may need - placeholder AWS credentials even though the proxy reissues upstream with its - own identity. -- `/healthz` proves provider registration and Flask availability, not upstream - credential validity. - -OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return -JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed -with the upstream status and are not byte-for-byte passthrough. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md b/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md deleted file mode 100644 index 8520c4a..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md +++ /dev/null @@ -1,81 +0,0 @@ -# Packaging and import roots - -Read this reference when an adapter imports nested source code, the source uses a -`src/` layout, or V031 reports an import-root problem. - -## What `/app` can import - -CanyonOS Core preserves project-relative paths in the image and starts Python at -`/app`. Without an editable install, Python resolves names rooted there: - -- `/app/tools.py` as `import tools` -- `/app/pkg/__init__.py` as `import pkg` -- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace - directories without `__init__.py` - -It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become -an import root first. - -## Detect support, do not infer it from release history - -Run: - -```bash -python /validate.py . -``` - -Read the `editable_install` capability. If it is unavailable and the original -import cannot resolve from `/app`, report a runtime capability blocker and stop. -Do not add a `sys.path` hack or relocate source files. - -## Root metadata is the trigger - -When editable install is supported, only packaging metadata at the **port root** -triggers `pip install -e .`: - -```text -port-root/pyproject.toml detected -port-root/source/pyproject.toml ignored as an install trigger -``` - -A nested source repository may remain untouched. Add minimal root scaffolding -that points package discovery at the existing source package: - -```toml -[build-system] -requires = ["setuptools>=64"] -build-backend = "setuptools.build_meta" - -[project] -name = "canyonos-port" -version = "0.0.0" -dependencies = [] - -[tool.setuptools.packages.find] -where = ["source/src"] -include = ["pkg*"] -namespaces = true -``` - -Set `where` and `include` from the actual tree and original import spelling. Do -not reference a README or license from this wrapper metadata; file sweeps differ -by runtime capability and a missing referenced file makes the image build fail. - -## Dependencies in nested metadata - -A nested `pyproject.toml` is not installed merely because its Python files are -copied. Keep the source declaration unchanged and repeat its runtime -distributions in each relevant config entry's `requirements` list. This is -compatibility scaffolding, not permission to drop, move, or reclassify declared -dependencies. - -If source metadata is already at the port root, do not create a wrapper. Its -project dependencies participate in the same resolver as config requirements. -Report declared-but-unused toolchain dependencies and their image cost; let the -owner decide whether source metadata should change. - -## Validation boundary - -`ventis build` owns packaging syntax and installation errors. `validate.py` -checks only whether adapter imports appear to require a nested root that the -runtime will not expose. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md deleted file mode 100644 index 94f7401..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md +++ /dev/null @@ -1,187 +0,0 @@ -# CanyonOS Core runtime contract - -The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the -CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for -Docker resources. - -Read this reference when implementing an adapter or explaining a validator -finding. Runtime-dependent behavior is expressed as capabilities; run -`validate.py` against the target environment instead of inferring support from -release history. - -## Project root and discovery - -`ventis build` uses the current working directory as the project root. - -| Input | Discovery | -|---|---| -| `agents/*.yaml` | direct yaml glob under the project root | -| `config/global_controller.yaml` | default config, overridable with `-c` | -| workflow | `workflow_file` on a `type: workflow` entry | -| policy | `policy.yaml` beside the selected config file | -| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | - -The config name, yaml `agent.name`, and entrypoint class name form one binding: - -```text -config entry name == yaml agent.name == entrypoint class name -``` - -A missing match may skip an image while the command continues, so inspect build -output and generated image tags. - -## Agent yaml and generated stubs - -The consumed yaml shape is: - -```yaml -agent: - name: ExampleAgent - functions: - - name: work - arguments: - - name: query - type: str - returns: - type: dict -``` - -Argument annotations are generated from bare names without adding imports. Use -builtins. Generated methods have no defaults, so every declared argument is -required at the stub call site. `returns` does not control runtime conversion; -it documents whether workflow code should parse the returned string. - -Stub destinations differ by runtime capability and entrypoint layout. Workflow -code in this port convention imports the generated class from -`agents.`. The validator checks that import against declarations -before the workflow image starts. - -## Agent loading and execution - -The local controller effectively performs: - -```python -module = load(entrypoint) -agent_class = getattr(module, configured_name) -agent = agent_class() -result = getattr(agent, method_name)(**args) -``` - -Consequences: - -- The class is module-level and named exactly as configured. -- Construction takes no arguments. -- Declared methods accept yaml argument names as keyword arguments. -- Methods are synchronous; this path does not await a coroutine. -- Dicts and lists are JSON-encoded before entering Redis; other results become - strings. -- A remote Future's `.value()` returns text, not the original Python object. - -Agent import and construction exceptions are caught by the controller. A failed -agent may still advertise healthy because health is written independently of -successful agent loading. That is why image probes import both the runtime and -the entrypoint explicitly. - -## Workflow execution - -The workflow file is executed, not imported. Therefore: - -- module-level code runs at container startup; -- `__name__ == "__main__"`; -- `deploy()` blocks in the web server; -- the workflow function runs once per request; -- its function name determines the REST route exposed by the compatibility - runtime. - -The deployment platform additionally expects `/main` with a `{query: string}` -body. This platform constraint is stricter than the underlying transport. - -Each stub method returns a Future immediately. `.value()` blocks. Dispatching -and resolving inside one comprehension serializes work without raising an -error; dispatch all calls first, then resolve them. - -The workflow container also starts runtime controller code and has its own -package resolution. Probe it independently from agent images. - -## Build context and collisions - -The runtime copies project files while preserving relative paths, then writes -shared runtime modules, generated stubs, and entrypoints into the image. Later -writes can shadow project files. - -Avoid root project modules named like runtime files, including: - -```text -future.py -ventis_context.py -local_controller.py -local_controller_frontend.py -redis_client.py -grpc_options.py -bedrock.py -deploy.py -session_logging.py -workflow_launcher.py -``` - -Also avoid a yaml basename that shadows a different source module imported by an -adapter. The validator checks deterministic flat-name collisions. - -File sweep and editable-install behavior are runtime capabilities. For nested -imports, follow [packaging.md](packaging.md). - -## Dependencies and protobuf - -Agent and workflow images include a small runtime dependency set. Config -`requirements` adds source-specific distributions. A malformed requirements -value can be normalized away while image generation continues; missing imports -then surface only when the agent loads. - -The build compiles gRPC Python stubs on the host and copies them into images. -The image resolver does not necessarily know the generated-code version. A -source dependency that constrains protobuf below the host generator version can -produce a green image build that dies on: - -```text -import local_controller -``` - -Always run that probe before probing the entrypoint. Treat a generated-code / -runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter -source dependencies silently. - -## Credentials capability - -When `env_file` capability is available, the top-level config path is resolved -against the project root and passed at container start. Hidden env files are not -copied into images. Invalid paths are deploy-preflight errors. - -When the capability is unavailable, declaring `env_file` has no effect. If the -source needs credentials, report the capability blocker rather than hardcoding -or vendoring a secret. - -A source that constructs its client at import time works only when credentials -are already in the container environment. Image entrypoint probes therefore use -the same env file as deployment. - -For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). - -## Policy and provider behavior - -No policy file means unrestricted service access. If a policy exists, it needs a -non-empty rules list. Rules are evaluated by specificity and first match; -services excluded from the selected rule fail after request acceptance. - -Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior -and remote networking are covered in [ec2.md](ec2.md). - -## Cleanup boundary - -Stopping foreground deploy normally invokes controller cleanup for recorded -containers and Redis. Hard kills and failures before resource registration may -leave resources behind. - -`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and -`docker_container/`. It does not remove containers or images. Remove exact -leftovers explicitly and preserve source, port scaffolding, and requested -evidence. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md deleted file mode 100644 index 361acf9..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md +++ /dev/null @@ -1,60 +0,0 @@ -# Troubleshooting - -Read this after a failed build, image probe, deploy, or request. For mechanisms, -read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, -read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). - -## Build or deploy stops early - -| Symptom | Likely cause | -|---|---| -| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | -| Two services produce one image | Config names collide after lowercase normalization | -| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | -| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | -| Replica conversion `TypeError` | `replicas` is not an integer | -| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | -| Port or container name already in use | A previous deployment did not complete cleanup | - -## Container exits or serves nothing - -| Symptom | Likely cause | -|---|---| -| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | -| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | -| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | -| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | -| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | -| Third-party module is missing | Distribution is absent from source metadata and config requirements | -| Stub import raises `NameError` | yaml argument type is not a bare builtin | -| Source module behaves like an empty stub | A generated stub basename shadowed the source module | -| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | - -## Request is accepted, then fails - -| Symptom | Likely cause | -|---|---| -| Unexpected keyword argument | yaml argument name differs from adapter parameter name | -| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | -| Unauthorized service | The first matching policy rule excludes that service | -| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | -| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | -| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | -| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | -| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | - -## Deployment platform endpoint - -| Symptom | Likely cause | -|---|---| -| 404 while workflow container is healthy | Workflow function is not named `main` | -| 400 before host receives request | Body is not the platform's `{query: string}` shape | -| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | - -## Cleanup - -| Symptom | Likely cause | -|---|---| -| `ventis clean` succeeds but containers remain | The command removes generated directories only | -| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | -| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/cli/.claude/skills/porting-to-canyonos-core/validate.py b/cli/.claude/skills/porting-to-canyonos-core/validate.py deleted file mode 100755 index 04baf37..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/validate.py +++ /dev/null @@ -1,1257 +0,0 @@ -#!/usr/bin/env python3 -"""Preflight the runtime traps that `ventis build` cannot see. - -This deliberately does not duplicate build-time validation such as malformed -YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those -checks. This script parses Python without importing it and catches failures that -otherwise stay hidden until a container loads an agent, starts a workflow, or -serves its first request. A replica is not evidence: the controller writes -`healthy` to Redis before `_load_agent` runs. - - python validate.py [project_dir] [-c config/global_controller.yaml] - [--json] [--strict] - -Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. - -Runtime capabilities vary across CanyonOS Core installations. This script probes -the importable `ventis` package directly. A capability-gated check reports -UNAVAILABLE when its behavior cannot be proven. -""" - -import argparse -import ast -import builtins -import json -import os -import re -import sys -from typing import ClassVar - -try: - import yaml -except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency - sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") - raise SystemExit(2) from None - - -DEFAULT_CONFIG_PATH = "config/global_controller.yaml" - -# Copied flat into every image over the swept project tree, so a project module -# landing flat under one of these names is overwritten. -# ventis/stub_generator.py generate_docker / generate_workflow_docker. -RUNTIME_FLAT_NAMES = frozenset( - { - "future.py", - "ventis_context.py", - "local_controller.py", - "local_controller_frontend.py", - "redis_client.py", - "grpc_options.py", - "gpu_metrics.py", - "bedrock.py", - "deploy.py", - "session_logging.py", - "workflow_launcher.py", - } -) - -# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. -BASE_AGENT_REQUIREMENTS = [ - "grpcio", - "grpcio-tools", - "redis", - "pyyaml", - "psutil", - "ipdb", - "ipython", - "boto3", -] -# Import name -> distribution name, for the handful where they differ and the -# mismatch would otherwise be reported as a missing requirement. -IMPORT_TO_DISTRIBUTION = { - "attr": "attrs", - "bs4": "beautifulsoup4", - "cv2": "opencv-python", - "dateutil": "python-dateutil", - "dotenv": "python-dotenv", - "grpc": "grpcio", - "grpc_tools": "grpcio-tools", - "jwt": "pyjwt", - "PIL": "pillow", - "psycopg": "psycopg", - "psycopg2": "psycopg2-binary", - "pydantic_settings": "pydantic-settings", - "sklearn": "scikit-learn", - "typing_extensions": "typing-extensions", - "yaml": "pyyaml", -} - -SECRET_PATTERNS = [ - (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), - (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), - (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), - (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), -] -SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) - -ERROR = "ERROR" -WARN = "WARN" -INFO = "INFO" - - -# ------------------------------------------------------------------ # -# Capabilities # -# ------------------------------------------------------------------ # -# -# Stable labels for behavior detected from the importable runtime. They contain -# no external development metadata. - -CAPABILITY_SOURCE = { - "env_file": "runtime env-file injection", - "editable_install": "editable project installation", - "sweeps_all_files": "full project-file sweep", - "stub_two_destinations": "flat and package stub destinations", -} - - -def probe_capabilities(): - """Ask the importable ventis package what it actually supports.""" - caps = dict.fromkeys(CAPABILITY_SOURCE, False) - caps["ventis"] = False - try: - from ventis import stub_generator - except Exception: # noqa: BLE001 - a broken install must not crash the check - return caps - - caps["ventis"] = True - caps["editable_install"] = hasattr(stub_generator, "_install_step") - caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") - caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") - - import importlib - - for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): - try: - module = importlib.import_module(module_name) - except Exception: # noqa: BLE001,S112 - the other path is the live one - continue - if hasattr(module, "resolve_env_file"): - caps["env_file"] = True - break - return caps - - -# ------------------------------------------------------------------ # -# YAML with line numbers # -# ------------------------------------------------------------------ # - - -class LineDict(dict): - """A mapping that remembers where it and each of its keys were written.""" - - line = 0 - key_lines: ClassVar[dict] = {} - - -class LineLoader(yaml.SafeLoader): - pass - - -def _construct_mapping(loader, node): - data = LineDict() - yield data - data.update(loader.construct_mapping(node, deep=False)) - data.line = node.start_mark.line + 1 - data.key_lines = { - key.value: key.start_mark.line + 1 - for key, _ in node.value - if isinstance(key, yaml.ScalarNode) - } - - -LineLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping -) - - -def line_of(mapping, key=None): - """The source line of `key` inside `mapping`, or of the mapping itself.""" - if not isinstance(mapping, LineDict): - return 0 - if key is not None: - return mapping.key_lines.get(key, mapping.line) - return mapping.line - - -def load_yaml(path): - """Parse `path`, returning (data, error). Never raises.""" - try: - with open(path, "r", encoding="utf-8") as handle: - return yaml.load(handle, Loader=LineLoader), None - except Exception as exc: # noqa: BLE001 - any parse failure is a finding - return None, str(exc) - - -# ------------------------------------------------------------------ # -# Findings # -# ------------------------------------------------------------------ # - - -class Report: - def __init__(self, project_dir, capabilities): - self.project_dir = project_dir - self.capabilities = capabilities - self.findings = [] - # A peer agent is imported by the name of its generated stub, which the - # build copies flat into every image. Those are not project modules and - # need no requirement. - self.stub_module_names = set() - - def add(self, check, level, path, line, summary, mechanism): - self.findings.append( - { - "check": check, - "level": level, - "path": self.rel(path) if path else "", - "line": line or 0, - "summary": summary, - "mechanism": mechanism, - } - ) - - def error(self, check, path, line, summary, mechanism): - self.add(check, ERROR, path, line, summary, mechanism) - - def warn(self, check, path, line, summary, mechanism): - self.add(check, WARN, path, line, summary, mechanism) - - def unavailable(self, check, summary): - self.add(check, INFO, "", 0, summary, "") - - def rel(self, path): - try: - return os.path.relpath(path, self.project_dir) - except ValueError: - return path - - def counts(self): - errors = sum(1 for f in self.findings if f["level"] == ERROR) - warnings = sum(1 for f in self.findings if f["level"] == WARN) - return errors, warnings - - -# ------------------------------------------------------------------ # -# Python source helpers # -# ------------------------------------------------------------------ # - - -def parse_python(path): - """AST for `path`, or (None, error). The port is never imported.""" - try: - with open(path, "r", encoding="utf-8") as handle: - source = handle.read() - except OSError as exc: - return None, str(exc) - try: - return ast.parse(source, filename=path), None - except SyntaxError as exc: - return None, f"{exc.msg} (line {exc.lineno})" - - -def find_class(tree, name): - for node in tree.body: - if isinstance(node, ast.ClassDef) and node.name == name: - return node - return None - - -def class_methods(class_node): - return { - node.name: node - for node in class_node.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - } - - -def parameter_names(func_node): - """Every parameter a caller can pass by keyword, minus self.""" - args = func_node.args - positional = [a.arg for a in args.posonlyargs + args.args] - if positional and positional[0] in ("self", "cls"): - positional = positional[1:] - return positional + [a.arg for a in args.kwonlyargs] - - -def required_parameters(func_node): - """Parameters with no default, minus self.""" - args = func_node.args - positional = [a.arg for a in args.posonlyargs + args.args] - if positional and positional[0] in ("self", "cls"): - positional = positional[1:] - if args.defaults: - positional = positional[: len(positional) - len(args.defaults)] - kwonly = [ - arg.arg - for arg, default in zip(args.kwonlyargs, args.kw_defaults) - if default is None - ] - return positional + kwonly - - -def toplevel_import_names(tree): - """Top-level package name of every import in the module, with line numbers.""" - names = {} - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - names.setdefault(alias.name.split(".")[0], node.lineno) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - names.setdefault(node.module.split(".")[0], node.lineno) - return names - - -# ------------------------------------------------------------------ # -# V006-V010 adapter failures hidden by _load_agent # -# ------------------------------------------------------------------ # - -BUILTIN_TYPE_NAMES = frozenset( - name - for name in dir(builtins) - if isinstance(getattr(builtins, name), type) and not name[0].isupper() -) - - -def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): - """V006 V007 V008 V009 V010.""" - name = agent_block["name"] - functions = agent_block.get("functions") or [] - check_argument_types(report, agent_yaml_path, functions) - - entrypoint = entry.get("entrypoint") - if not entrypoint: - return - entrypoint_path = os.path.join(project_dir, entrypoint) - if not os.path.isfile(entrypoint_path): - return - - tree, error = parse_python(entrypoint_path) - if tree is None: - report.error( - "V006", - entrypoint_path, - 0, - f"the entrypoint does not parse: {error}", - "_load_agent exec_module's it and swallows the exception; the first " - "request answers 'No agent loaded'.", - ) - return - - class_node = find_class(tree, name) - if class_node is None: - classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] - found = ", ".join(classes) if classes else "no classes at all" - report.error( - "V006", - entrypoint_path, - 1, - f"no class named `{name}` at module level (found: {found})", - "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " - "the AttributeError. The class name must equal agent.name exactly.", - ) - return - - methods = class_methods(class_node) - check_constructor(report, entrypoint_path, name, methods) - - for func in functions: - if not isinstance(func, dict) or not isinstance(func.get("name"), str): - continue - check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) - - -def check_argument_types(report, agent_yaml_path, functions): - """V010 -- the type string is pasted into an ast.Name, never checked.""" - for func in functions: - if not isinstance(func, dict): - continue - for arg in func.get("arguments") or []: - if not isinstance(arg, dict) or "type" not in arg: - continue - declared = arg.get("type") - if not isinstance(declared, str): - continue # stub generation reports malformed type values - if declared in BUILTIN_TYPE_NAMES: - continue - report.error( - "V010", - agent_yaml_path, - line_of(arg, "type"), - f"`type: {declared}` is not a builtin", - "stub_generator pastes it verbatim into the generated " - "annotation, and the stub module imports only Future and " - "inspect. Anything else raises NameError when the stub is " - "imported -- after a green build. Use str int float bool dict " - "list.", - ) - - -def check_constructor(report, entrypoint_path, name, methods): - """V007 -- _load_agent calls agent_class() with no arguments.""" - init = methods.get("__init__") - if init is None: - return - required = required_parameters(init) - if required: - report.error( - "V007", - entrypoint_path, - init.lineno, - f"`{name}.__init__` requires {', '.join(required)}", - "_load_agent calls agent_class() with no arguments; the TypeError is " - "swallowed and the first request answers 'No agent loaded'. Read " - "configuration from the environment inside __init__ instead.", - ) - - -def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): - """V008 V009.""" - func_name = func["name"] - method = methods.get(func_name) - if method is None: - report.error( - "V008", - entrypoint_path, - 0, - f"`{class_name}` has no method `{func_name}`", - "The yaml declares it, so callers get a stub for it; the controller " - f"then answers \"Agent {class_name} has no method '{func_name}'\".", - ) - return - - # V009 -- nothing on the execution path awaits. - if isinstance(method, ast.AsyncFunctionDef): - report.error( - "V009", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` is `async def`", - "The executor calls method(**args) with no await, so Redis receives " - "''. Keep the signature synchronous and call " - "asyncio.run(...) inside the body.", - ) - - # V008 -- the controller calls method(**args) with the yaml's names. - declared = [ - arg["name"] - for arg in func.get("arguments") or [] - if isinstance(arg, dict) and isinstance(arg.get("name"), str) - ] - actual = parameter_names(method) - required = required_parameters(method) - - missing = [arg for arg in declared if arg not in actual] - if missing: - report.error( - "V008", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` has no parameter " - f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", - "LocalController does method(**args) with the yaml's argument names. " - "A mismatch is TypeError: unexpected keyword argument, at request " - f"time. See {os.path.basename(agent_yaml_path)}.", - ) - - unfilled = [arg for arg in required if arg not in declared] - if unfilled: - report.error( - "V008", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " - "the yaml does not declare", - "Only declared arguments are ever sent, and the generated stub gives " - "none of them a default. Declare them in the yaml or default them " - "in the signature.", - ) - - - -# ------------------------------------------------------------------ # -# V016-V018 the workflow # -# ------------------------------------------------------------------ # - - -def check_stub_imports(report, workflow_path, tree, stub_classes): - """V023 -- the workflow must import a stub as `from agents. import `. - - The build copies each stub to exactly one path, and for the workflow image - that path is agents/.py. Two ways of writing this line fail, and - the project walks you into both: the flat form is what examples/ uses, and - the class name is the one `ventis build` prints, which is not the one it - writes. - """ - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - base = alias.name.split(".")[0] - if base in stub_classes: - report.error( - "V023", workflow_path, node.lineno, - f"`import {alias.name}` -- the stub is at " - f"agents/{base}.py, not flat", - "The build copies a stub to one path, and for the " - "workflow that path is under agents/. This is a " - "ModuleNotFoundError the moment the workflow runs. " - f"Write `from agents.{base} import {stub_classes[base]}`.", - ) - continue - - if not isinstance(node, ast.ImportFrom) or not node.module: - continue - - module = node.module - if module in stub_classes: - report.error( - "V023", workflow_path, node.lineno, - f"`from {module} import ...` -- the stub is at " - f"agents/{module}.py, not flat", - "The build copies a stub to one path, and for the workflow " - "that path is under agents/. The flat form is what this " - "repository's own examples use and it raises " - "ModuleNotFoundError in the workflow image. Write " - f"`from agents.{module} import {stub_classes[module]}`.", - ) - continue - - if not module.startswith("agents."): - continue - base = module.split(".", 1)[1] - expected = stub_classes.get(base) - if expected is None: - continue - for alias in node.names: - if alias.name == expected: - continue - if alias.name == f"{expected}Stub": - report.error( - "V023", workflow_path, node.lineno, - f"`{alias.name}` is the name the build prints, not the " - f"class it writes", - "generate_agent_stub sets class_name = agent_config['name'] " - "and then recomputes it with a 'Stub' suffix for the log " - "line only. The message names a class that does not exist; " - f"the class is `{expected}`.", - ) - else: - report.error( - "V023", workflow_path, node.lineno, - f"`{alias.name}` is not a class the stub for {base} defines", - f"The stub's class carries the agent's own name: `{expected}`.", - ) - - -def check_workflow(report, workflow_path, stub_classes=None): - """V016 V017 V018 V023.""" - tree, error = parse_python(workflow_path) - if tree is None: - report.error("V016", workflow_path, 0, f"does not parse: {error}", "") - return - - main = None - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( - node.name == "main" - ): - main = node - break - - if main is None: - defined = [ - n.name - for n in tree.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) - ] - found = ", ".join(defined) if defined else "no top-level functions" - report.error( - "V016", - workflow_path, - 1, - f"no top-level function named `main` (found: {found})", - "CanyonOS Core serves POST /, but the deployment platform's " - "test endpoint posts to a hardcoded /main. A differently named " - "workflow builds, deploys and stays unreachable -- 404, container " - "healthy.", - ) - else: - check_main_signature(report, workflow_path, main) - - if stub_classes: - check_stub_imports(report, workflow_path, tree, stub_classes) - - # V016 -- deploy() is what starts Flask. - if not any( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "deploy" - for node in ast.walk(tree) - ): - report.error( - "V016", - workflow_path, - 1, - "the workflow never calls `deploy(...)`", - "workflow_launcher.py exec's this file and nothing else starts the " - "HTTP server; the container comes up serving nothing.", - ) - - check_main_guard(report, workflow_path, tree) - check_fused_fanout(report, workflow_path, tree) - - -def check_main_signature(report, workflow_path, main): - """V016 -- the platform sends exactly {"query": ...}.""" - if isinstance(main, ast.AsyncFunctionDef): - report.error( - "V016", - workflow_path, - main.lineno, - "`main` is `async def`", - "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " - "no await; the response body would be a coroutine repr.", - ) - params = parameter_names(main) - if not params: - report.error( - "V016", - workflow_path, - main.lineno, - "`main` takes no arguments", - 'The platform posts {"query": "..."} and deploy() splats the ' - "body in as kwargs -- TypeError on every request.", - ) - return - if params[0] != "query": - report.error( - "V016", - workflow_path, - main.lineno, - f"`main`'s first parameter is `{params[0]}`, not `query`", - "The platform's body schema is strictly validated as " - "{query: string}; any other key is rejected with 400 in the control " - "plane, before the request reaches the host.", - ) - extra = [p for p in required_parameters(main) if p != "query"] - if extra: - report.error( - "V016", - workflow_path, - main.lineno, - f"`main` requires {', '.join(extra)} beyond `query`", - "Only `query` is ever sent, so every other parameter needs a " - "default or the call raises on every request. Pack richer input " - "into `query`.", - ) - - -def check_main_guard(report, workflow_path, tree): - """V017 -- the workflow is exec'd, so __name__ == "__main__".""" - for node in tree.body: - if not isinstance(node, ast.If): - continue - test = node.test - if ( - isinstance(test, ast.Compare) - and isinstance(test.left, ast.Name) - and test.left.id == "__name__" - and any( - isinstance(c, ast.Constant) and c.value == "__main__" - for c in test.comparators - ) - ): - report.error( - "V017", - workflow_path, - node.lineno, - '`if __name__ == "__main__":` block in the workflow', - "workflow_launcher.py runs exec(open().read()), so " - "__name__ IS '__main__' here and this block executes in " - "production, at container start.", - ) - - -def check_fused_fanout(report, workflow_path, tree): - """V018 -- .value() blocks, so dispatching and resolving in one - comprehension runs the fan-out one call at a time.""" - comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) - for node in ast.walk(tree): - if not isinstance(node, comprehensions + (ast.DictComp,)): - continue - elements = ( - [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] - ) - for element in elements: - for inner in ast.walk(element): - if ( - isinstance(inner, ast.Call) - and isinstance(inner.func, ast.Attribute) - and inner.func.attr == "value" - and isinstance(inner.func.value, ast.Call) - ): - report.error( - "V018", - workflow_path, - node.lineno, - "one comprehension both dispatches a call and resolves " - "it with .value()", - ".value() blocks, so each call completes before the " - "next is dispatched. It does not error -- the fan-out " - "is just silently serial, and with it the reason to be " - "on CanyonOS Core. Dispatch every call first, then resolve: " - "futures = [a.work(i) for i in items] then " - "[f.value() for f in futures].", - ) - return - - -# ------------------------------------------------------------------ # -# V019-V020 what the copy order overwrites # -# ------------------------------------------------------------------ # - - -def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): - """V019 V020 -- later copies land on earlier ones at the context root.""" - for entry in sorted(os.listdir(project_dir)): - path = os.path.join(project_dir, entry) - if not os.path.isfile(path) or not entry.endswith(".py"): - continue - - # V019 -- the shared runtime is copied flat, after the project sweep. - if entry in RUNTIME_FLAT_NAMES: - report.error( - "V019", - path, - 1, - f"a project module named `{entry}` sits at the project root", - "The shared CanyonOS Core runtime is copied flat into the image after " - "the project sweep, so this file is overwritten by CanyonOS Core's own " - f"{entry}. Rename it or move it into a package directory.", - ) - - # V020 -- a stub is copied flat under its yaml's basename. - entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} - for yaml_path in yaml_paths: - stem = os.path.splitext(os.path.basename(yaml_path))[0] - module = f"{stem}.py" - if module in entrypoint_basenames: - continue # the entrypoint is copied last and wins its flat name back - candidate = os.path.join(project_dir, module) - if os.path.isfile(candidate): - report.error( - "V020", - candidate, - 1, - f"`{os.path.basename(yaml_path)}` generates a stub that lands on " - f"`{module}`", - "The yaml's basename names the stub, and the stub is copied flat " - "over the swept tree. Anything importing this module inside the " - "container gets the generated stub instead of the real code. " - "Rename the yaml to match its own entrypoint.", - ) - - -# ------------------------------------------------------------------ # -# V030-V031 capability-gated rules # -# ------------------------------------------------------------------ # - - -def check_env_file(report, config, config_path, project_dir): - """V030 -- gated on detected env-file injection support.""" - declared = config.get("env_file") - supported = report.capabilities.get("env_file") - - if not supported: - if declared: - report.error( - "V030", - config_path, - line_of(config, "env_file"), - f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", - "No resolve_env_file in the importable ventis package, so the " - "key is silently dropped and the container answers a provider " - "credential error on the first request. This port requires the " - "`env_file` runtime capability.", - ) - else: - report.unavailable( - "V030", - "env_file is not supported by the importable `ventis` runtime. " - "Credentials have no declared path into a container on this tree.", - ) - return - - if not declared: - report.warn( - "V030", - config_path, - line_of(config), - "no `env_file:` in the config", - "Only runtime-managed VENTIS_* variables are guaranteed without it. " - "If the source reads credentials from the environment, the first " - "request fails on a provider error.", - ) - return - - # Path existence and readability are deploy-preflight checks. Do not - # duplicate them here. - - -def check_import_root(report, project_dir, entrypoint_paths): - """V031 -- gated on detected editable-install support.""" - supported = report.capabilities.get("editable_install") - has_metadata = any( - os.path.isfile(os.path.join(project_dir, name)) - for name in ("pyproject.toml", "setup.py", "setup.cfg") - ) - - non_flat = [] - for path in entrypoint_paths: - tree, _ = parse_python(path) - if tree is None: - continue - for name, lineno in toplevel_import_names(tree).items(): - if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: - continue - if _resolves_flat(project_dir, name): - continue - location = _resolves_nested(project_dir, name) - if location: - non_flat.append((path, lineno, name, location)) - - if not supported: - report.unavailable( - "V031", - "the editable install (`-e .`) is not supported by the importable " - "`ventis` runtime. Only names rooted at /app import inside a container.", - ) - for path, lineno, name, location in non_flat: - report.error( - "V031", - path, - lineno, - f"`import {name}` resolves to {location}, which is not at the " - "project root", - "sys.path[0] is /app and this CanyonOS Core runs no editable install, " - "so only modules swept to the root import. The adapter raises " - "ModuleNotFoundError inside _load_agent and the first request " - "answers 'No agent loaded'.", - ) - return - - if non_flat and not has_metadata: - for path, lineno, name, location in non_flat: - report.error( - "V031", - path, - lineno, - f"`import {name}` resolves to {location}, and the project root " - "has no packaging metadata", - "A pyproject.toml, setup.py or setup.cfg at the port root is " - "what adds `-e .`; metadata nested inside the untouched source " - "tree is ignored. Add minimal root metadata pointing at the " - "existing package directory. Without it the install is skipped " - "silently.", - ) - - -def _resolves_flat(project_dir, name): - """Whether Python can resolve `name` with /app as its import root. - - A directory does not need __init__.py: PEP 420 namespace packages resolve - from sys.path just like regular packages. - """ - return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( - os.path.join(project_dir, name) - ) - - -def _resolves_nested(project_dir, name): - """Where below /app `name` lives but cannot resolve as a top-level name.""" - for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] - if root == project_dir: - continue - if f"{name}.py" in files: - return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) - if name in dirs: - return os.path.relpath(os.path.join(root, name), project_dir) - return None - - -# ------------------------------------------------------------------ # -# W003, W006 secrets and imports a green build does not reject # -# ------------------------------------------------------------------ # - - -def check_secrets(report, port_paths): - """W003 -- env_file is the way in; nothing else is.""" - for path in port_paths: - try: - with open(path, "r", encoding="utf-8") as handle: - lines = handle.read().splitlines() - except OSError: - continue - for number, line in enumerate(lines, start=1): - for pattern, description in SECRET_PATTERNS: - if pattern.search(line): - report.warn( - "W003", - path, - number, - f"this line looks like {description}", - "Never put a secret in the source tree or the build " - "context. The build sweeps the project into every " - "image.", - ) - break - - tree, _ = parse_python(path) - if tree is None: - continue - for node in ast.walk(tree): - if not isinstance(node, ast.Assign): - continue - if not ( - isinstance(node.value, ast.Constant) - and isinstance(node.value.value, str) - and node.value.value.strip() - ): - continue - for target in node.targets: - if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): - report.warn( - "W003", - path, - node.lineno, - f"`{target.id}` is assigned a literal string", - "Read it from the environment instead; the build sweeps " - "this file into every image.", - ) - - -def _pyproject_dependencies(project_dir): - """What `-e .` installs alongside `requirements:`, or None if unreadable. - - None and the empty set mean different things here: empty means the project - declares no dependencies, None means we could not find out -- a setup.py, or - a tomllib this interpreter does not have. The caller must not treat the - second as the first, or it warns about imports the install would satisfy. - """ - path = os.path.join(project_dir, "pyproject.toml") - if not os.path.isfile(path): - return None - try: - import tomllib - except ImportError: # < 3.11 - return None - try: - with open(path, "rb") as handle: - data = tomllib.load(handle) - except Exception: # noqa: BLE001 - malformed metadata is uv's error to give - return None - deps = (data.get("project") or {}).get("dependencies") - if not isinstance(deps, list): - return None - return {_normalize_distribution(d) for d in deps if isinstance(d, str)} - - -def check_requirements_coverage( - report, project_dir, entry, entrypoint_path, config_path -): - """W006 -- an import the container cannot satisfy.""" - tree, _ = parse_python(entrypoint_path) - if tree is None: - return - - declared = { - _normalize_distribution(item) - for item in (entry.get("requirements") or []) - if isinstance(item, str) - } - - # Where the editable install exists, `-e .` resolves the project's own - # [project.dependencies] in the same pass as `requirements:`. Warning about - # those is a false positive, and a false warning about a dependency is worse - # than none: it teaches the reader to dismiss this check. - editable = report.capabilities.get("editable_install") - metadata = any( - os.path.isfile(os.path.join(project_dir, name)) - for name in ("pyproject.toml", "setup.py", "setup.cfg") - ) - unreadable_metadata = False - if editable and metadata: - project_deps = _pyproject_dependencies(project_dir) - if project_deps is None: - unreadable_metadata = True - else: - declared |= project_deps - base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} - stdlib = getattr(sys, "stdlib_module_names", frozenset()) - - for name, lineno in sorted(toplevel_import_names(tree).items()): - if name in stdlib or name == "ventis": - continue - # Provided by the image itself: the shared runtime is copied flat, and - # every agents/*.yaml generates a stub that is copied flat too. - if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: - continue - if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): - continue - distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) - if distribution in base or distribution in declared: - continue - if unreadable_metadata: - mechanism = ( - "The container installs the base list, `requirements:`, and -- " - "since this project declares packaging metadata -- whatever " - "`-e .` resolves from it. That metadata could not be read here, " - f"so if it already requires `{name}` this line is noise; " - "otherwise it is a ModuleNotFoundError inside _load_agent and " - "'No agent loaded' on the first request." - ) - else: - mechanism = ( - "The container installs the base list plus `requirements:` and " - "nothing else, so this is a ModuleNotFoundError inside " - "_load_agent and 'No agent loaded' on the first request. If the " - f"distribution is named something other than `{name}`, declare " - f"that name in {report.rel(config_path)}." - ) - report.warn( - "W006", - entrypoint_path, - lineno, - f"`import {name}` is in neither the runtime's base list nor this " - "entry's `requirements:`", - mechanism, - ) - - -def _normalize_distribution(name): - return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( - "_", "-" - ) - - -# ------------------------------------------------------------------ # -# Driver # -# ------------------------------------------------------------------ # - - -def validate(project_dir, config_path, capabilities): - """Inspect only failures hidden behind a successful image build.""" - report = Report(project_dir, capabilities) - - # The build owns config/YAML syntax and shape validation. We read only enough - # valid structure to locate code for the deeper checks below. - config, error = load_yaml(config_path) - if error is not None or not isinstance(config, dict): - report.unavailable( - "BUILD", - "runtime preflight skipped because the config cannot be read; " - "ventis build owns and reports this error.", - ) - return report - - import glob - - yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) - report.stub_module_names = { - os.path.splitext(os.path.basename(path))[0] for path in yaml_paths - } - - agents_by_name = {} - stub_classes = {} - for path in yaml_paths: - data, yaml_error = load_yaml(path) - agent = data.get("agent") if isinstance(data, dict) else None - name = agent.get("name") if isinstance(agent, dict) else None - if yaml_error is not None or not isinstance(name, str): - continue # ventis build reports malformed agent declarations - agents_by_name[name] = (path, agent) - stub_classes[os.path.splitext(os.path.basename(path))[0]] = name - - entries = config.get("agents") - if not isinstance(entries, list): - report.unavailable( - "BUILD", - "runtime preflight skipped because `agents:` is not a list; " - "ventis build owns and reports this error.", - ) - return report - - entrypoints = [] - for entry in entries: - if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": - continue - name = entry.get("name") - entrypoint = entry.get("entrypoint") - if isinstance(entrypoint, str): - entrypoints.append(entrypoint) - if name in agents_by_name: - yaml_path, agent_block = agents_by_name[name] - check_adapter(report, yaml_path, agent_block, entry, project_dir) - entrypoint_path = os.path.join(project_dir, entrypoint or "") - if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): - check_requirements_coverage( - report, project_dir, entry, entrypoint_path, config_path - ) - - for entry in entries: - if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": - continue - workflow_file = entry.get("workflow_file") - if not isinstance(workflow_file, str): - continue - workflow_path = os.path.join(project_dir, workflow_file) - if os.path.isfile(workflow_path): - check_workflow(report, workflow_path, stub_classes) - - # These survive a green build and otherwise surface only in a container or - # on its first request. - check_flat_collisions(report, project_dir, yaml_paths, entrypoints) - check_env_file(report, config, config_path, project_dir) - - entrypoint_paths = [ - os.path.join(project_dir, e) - for e in entrypoints - if os.path.isfile(os.path.join(project_dir, e)) - ] - check_import_root(report, project_dir, entrypoint_paths) - - port_paths = list(entrypoint_paths) - for entry in entries: - if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): - candidate = os.path.join(project_dir, entry["workflow_file"]) - if os.path.isfile(candidate): - port_paths.append(candidate) - - # Secret detection remains because a green image build would permanently - # bake the credential into every image. - check_secrets(report, port_paths) - return report - - - -# ------------------------------------------------------------------ # -# Output # -# ------------------------------------------------------------------ # - -LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} - - -def _wrap(text, width, indent): - words = text.split() - lines = [] - current = "" - for word in words: - candidate = f"{current} {word}".strip() - if len(candidate) + len(indent) > width and current: - lines.append(indent + current) - current = word - else: - current = candidate - if current: - lines.append(indent + current) - return lines - - -def print_report(report, project_dir): - caps = report.capabilities - if not caps.get("ventis"): - print("ventis is not importable here -- capability-gated rules are") - print("reported UNAVAILABLE rather than checked.\n") - else: - print("CanyonOS Core capabilities detected:") - for key, source in CAPABILITY_SOURCE.items(): - mark = "yes" if caps.get(key) else "no " - print(f" {mark} {key:<22} {source}") - print() - - findings = sorted( - report.findings, - key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), - ) - for finding in findings: - where = finding["path"] - if where and finding["line"]: - where = f"{where}:{finding['line']}" - header = f"{finding['check']} {finding['level']:<5}" - print(f"{header} {where}" if where else header) - for line in _wrap(finding["summary"], 78, " "): - print(line) - if finding["mechanism"]: - for line in _wrap(finding["mechanism"], 78, " "): - print(line) - print() - - errors, warnings = report.counts() - if not findings: - print(f"{project_dir}: clean.") - return - print(f"{errors} error(s), {warnings} warning(s).") - - -def main(argv=None): - parser = argparse.ArgumentParser( - description="Check a CanyonOS Core port against the rules in SKILL.md." - ) - parser.add_argument( - "project_dir", nargs="?", default=".", help="the port's project root" - ) - parser.add_argument( - "-c", - "--config", - default=DEFAULT_CONFIG_PATH, - help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", - ) - parser.add_argument("--json", action="store_true", help="emit findings as JSON") - parser.add_argument( - "--strict", action="store_true", help="fail on warnings as well as errors" - ) - args = parser.parse_args(argv) - - project_dir = os.path.abspath(args.project_dir) - config_path = ( - args.config - if os.path.isabs(args.config) - else os.path.join(project_dir, args.config) - ) - - capabilities = probe_capabilities() - report = validate(project_dir, config_path, capabilities) - errors, warnings = report.counts() - - if args.json: - print( - json.dumps( - { - "project_dir": project_dir, - "capabilities": capabilities, - "errors": errors, - "warnings": warnings, - "findings": report.findings, - }, - indent=2, - ) - ) - else: - print_report(report, report.rel(project_dir) or project_dir) - - if errors or (args.strict and warnings): - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From ced549d713f04931915886662978cfd3c7eb75ff Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 15:49:52 -0700 Subject: [PATCH 28/31] cli --- cli/README.md | 13 +- cli/canyonos/build.py | 205 +++++++++++++++++++ cli/canyonos/clean.py | 24 +-- cli/canyonos/config.py | 29 ++- cli/canyonos/constants.py | 53 ++++- cli/canyonos/dashboard.compose.yml | 11 +- cli/canyonos/dashboard_stack.py | 130 ++---------- cli/canyonos/deploy.py | 110 +++++++--- cli/canyonos/doctor.py | 96 +++++++++ cli/canyonos/gc.py | 83 ++++++++ cli/canyonos/init.py | 108 +++++++++- cli/canyonos/integrate.py | 82 -------- cli/canyonos/logs.py | 24 +-- cli/canyonos/quit.py | 18 +- cli/canyonos/serve.py | 4 +- cli/canyonos/stop.py | 37 +--- cli/canyonos/sync.py | 18 +- cli/canyonos/test.py | 173 ++++++++++++++++ cli/cli.py | 190 ++++++----------- cli/utils/help_screen.py | 60 ++++++ cli/utils/tui.py | 2 - {cli/tests => tests}/test_dashboard_stack.py | 0 22 files changed, 987 insertions(+), 483 deletions(-) create mode 100644 cli/canyonos/build.py create mode 100644 cli/canyonos/doctor.py create mode 100644 cli/canyonos/gc.py delete mode 100644 cli/canyonos/integrate.py create mode 100644 cli/canyonos/test.py create mode 100644 cli/utils/help_screen.py rename {cli/tests => tests}/test_dashboard_stack.py (100%) diff --git a/cli/README.md b/cli/README.md index de66cd0..76cd4e1 100644 --- a/cli/README.md +++ b/cli/README.md @@ -4,11 +4,18 @@ Serves as a thin API layer, connecting to the global controller container. ## Serve -`canyonos serve -c config/global_controller.yaml` starts the local CanyonOS dashboard. It reads -`database.url` from the config and writes only `CANYONOS_`-prefixed settings to the current `.env`, -leaving other lines unchanged. +`canyonos serve` starts the local CanyonOS dashboard against the Postgres bundled in its compose +stack — it reads no project config, so it takes no arguments. It writes only `CANYONOS_`-prefixed +settings into the current directory's `.env`, leaving every other line unchanged. +## Requirements +Need a coding agent(Claude Code, Codex, Cursor) +Need uv or pip +Need docker and docker compose +If you have a workflow running, and want to make a config change, canyonos config automatically would reload the project with your config. If you change the workflow files itself though and want the changes to take effect, you need to redeploy from scratch, running canyonos build for good measure + +# Use: canyonos -h ### To Republish to PyPi ```Terminal diff --git a/cli/canyonos/build.py b/cli/canyonos/build.py new file mode 100644 index 0000000..f5292aa --- /dev/null +++ b/cli/canyonos/build.py @@ -0,0 +1,205 @@ +""" +Logic for `canyonos build`: install the CanyonOS skill on a coding agent, +then launch that agent with a prompt to apply it to the current project. +""" + +import os +import shutil +import subprocess +import tarfile +import tempfile +import urllib.request + +from rich.console import Console + +from utils.tui import select_menu + +SKILL_OWNER = "CanyonCodeCoreAI" +SKILL_REPO = "canyoncodecore" +# The .car-aware skill lives only on this branch; the copies on main and every +# other branch are the older flat-layout `porting-to-canyonos-core`. Repoint at +# main once this merges -- and rename SKILL_NAME with it, since the two +# variants declare different `name:` frontmatter. +SKILL_REF = "nickhuo/porting-skill-car-layout" +SKILL_NAME = "porting-to-canyonos" +SKILL_PATH = f".claude/skills/{SKILL_NAME}" + +REPO_URL = f"https://github.com/{SKILL_OWNER}/{SKILL_REPO}" +TREE_URL = f"{REPO_URL}/tree/{SKILL_REF}/{SKILL_PATH}" +TARBALL_URL = f"https://codeload.github.com/{SKILL_OWNER}/{SKILL_REPO}/tar.gz/refs/heads/{SKILL_REF}" + +# The porting skill emits no otel config; without this the dashboard stays empty. +OTEL_BLOCK = """otel: + destinations: + - name: local + protocol: http + endpoint: http://host.docker.internal:3000/v1/traces + headers: {}""" + +BUILD_PROMPT = ( + f"Use the CanyonOS {SKILL_NAME} skill to convert the codebase in this directory to a canyonos-compatable format. No changes should be made to the current files, but all modifications should be put into a new .car folder." + "\n\nFinally, add the following block verbatim to the generated config/global_controller.yaml," + " at the top level as a sibling of `agents:`. Copy it exactly -- `protocol` must be http, and" + " the endpoint must keep the /v1/traces path:\n\n" + OTEL_BLOCK +) + +AGENTS = { + "claude": { + "label": "Claude Code", + "cli": "claude", + # Claude Code auto-loads project-local skills from here. The leaf name + # must match the skill's own `name:` frontmatter or it won't resolve. + "skill_dir": SKILL_PATH, + }, + "codex": { + "label": "Codex", + "cli": "codex", + # Codex only auto-loads skills from the user's home directory, not per-project. + "skill_dir": os.path.expanduser(f"~/.codex/skills/{SKILL_NAME}"), + }, +} + + +def prompt_agent(): + options = [(key, spec["label"]) for key, spec in AGENTS.items()] + return select_menu(options, title="Which coding agent do you want to build on?") + + +def _replace_dir(source, dest): + """Move `source` onto `dest`, replacing whatever was there.""" + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + if os.path.isdir(dest): + shutil.rmtree(dest) + shutil.move(source, dest) + + +def _fetch_with_git(dest): + """Sparse-checkout just the skill path -- no full-repo download, no Node.""" + if not shutil.which("git"): + return False + + with tempfile.TemporaryDirectory() as tmp: + clone = os.path.join(tmp, "repo") + cloned = subprocess.run( + ["git", "clone", "--depth", "1", "--filter=blob:none", "--sparse", + "--branch", SKILL_REF, REPO_URL, clone], + capture_output=True, + ) + if cloned.returncode != 0: + return False + + sparse = subprocess.run( + ["git", "-C", clone, "sparse-checkout", "set", SKILL_PATH], + capture_output=True, + ) + skill = os.path.join(clone, SKILL_PATH) + if sparse.returncode != 0 or not os.path.isdir(skill): + return False + + _replace_dir(skill, dest) + return True + + +def _fetch_with_tarball(dest): + """Stdlib-only fallback: pull the ref's tarball and keep the skill members. + + Needs no external tool at all, at the cost of downloading the whole repo. + """ + prefix = f"{SKILL_PATH}/" + with tempfile.TemporaryDirectory() as tmp: + archive = os.path.join(tmp, "repo.tar.gz") + try: + with urllib.request.urlopen(TARBALL_URL, timeout=60) as response: + with open(archive, "wb") as out: + shutil.copyfileobj(response, out) + except OSError: + return False + + staged = os.path.join(tmp, "skill") + found = False + with tarfile.open(archive, "r:gz") as tar: + for member in tar.getmembers(): + # Drop the archive's own top-level directory, whose name + # depends on how GitHub mangles the ref. + _, _, path = member.name.partition("/") + if not path.startswith(prefix) or not member.isfile(): + continue + relative = os.path.relpath(path, SKILL_PATH) + target = os.path.join(staged, relative) + # Never let an archive entry write outside the staging dir. + if not os.path.abspath(target).startswith(os.path.abspath(staged) + os.sep): + continue + extracted = tar.extractfile(member) + if extracted is None: + continue + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "wb") as out: + shutil.copyfileobj(extracted, out) + found = True + + if not found: + return False + _replace_dir(staged, dest) + return True + + +def _fetch_with_npx(dest): + """Last resort, and the only strategy that needs Node.""" + if not shutil.which("npx"): + return False + # -f overwrites an existing skill dir; without it gitpick exits 1 when the + # target already exists and is non-empty (e.g. re-running `build`). + return subprocess.run( + ["npx", "-y", "gitpick", "-f", TREE_URL, dest], capture_output=True + ).returncode == 0 + + +FETCH_STRATEGIES = ( + ("git", _fetch_with_git), + ("tarball", _fetch_with_tarball), + ("npx", _fetch_with_npx), +) + + +def install_skill(agent, console): + """Fetch the skill into the agent's skill dir. Returns True on success.""" + dest = AGENTS[agent]["skill_dir"] + for name, fetch in FETCH_STRATEGIES: + try: + if fetch(dest): + console.print(f"Fetched the CanyonOS skill via {name}.") + return True + except OSError: + pass + console.print(f"[dim]{name} fetch unavailable, trying the next option...[/dim]") + + console.print( + f"Could not fetch the CanyonOS skill from {TREE_URL}.\n" + "Install git or Node, or check network access, then run `canyonos doctor`." + ) + return False + + +def launch_agent(agent, prompt): + spec = AGENTS[agent] + if not shutil.which(spec["cli"]): + print(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") + return + # No check=True: the agent exiting non-zero (including the user quitting it) + # is an ordinary outcome, not something to raise a traceback over. + subprocess.run([spec["cli"], prompt]) + + +def run_build(): + console = Console() + agent = prompt_agent() + if agent is None: + console.print("Cancelled.") + return + + console.print(f"Installing CanyonOS skill for {AGENTS[agent]['label']}...") + if not install_skill(agent, console): + return + + console.print(f"Launching {AGENTS[agent]['label']}...") + launch_agent(agent, BUILD_PROMPT) diff --git a/cli/canyonos/clean.py b/cli/canyonos/clean.py index aabf105..d7cc6f5 100644 --- a/cli/canyonos/clean.py +++ b/cli/canyonos/clean.py @@ -1,7 +1,5 @@ """ -Remove generated stubs, gRPC files, and Docker build contexts. - -Ported directly over from canyonos, moving the logic into here. +Logic for `canyonos clean`: remove the generated .car artifact directory. """ import os @@ -9,20 +7,12 @@ def run_clean(): - project_dir = os.getcwd() - - paths_to_clean = [ - os.path.join(project_dir, "stubs"), - os.path.join(project_dir, "grpc_stubs"), - os.path.join(project_dir, "docker_container"), - ] + car_dir = os.path.join(os.getcwd(), ".car") - for path in paths_to_clean: - if os.path.exists(path): - print(f"Cleaning {path}...") - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.remove(path) + if not os.path.isdir(car_dir): + print("Nothing to clean, no .car folder in root") + return + print(f"Cleaning {car_dir}...") + shutil.rmtree(car_dir) print("Clean complete.") diff --git a/cli/canyonos/config.py b/cli/canyonos/config.py index 226312e..25d2d3d 100644 --- a/cli/canyonos/config.py +++ b/cli/canyonos/config.py @@ -7,9 +7,8 @@ import yaml from rich.console import Console from rich.table import Table -from ruamel.yaml import YAML -from canyonos.constants import default_config_path +from canyonos.constants import default_config_path, round_trip_yaml from canyonos.theme import GREEN, WHITE from utils.tui import DELETE_ACTION, QUIT_ACTION, select_menu @@ -99,12 +98,19 @@ def _kv_table(title, data): return table -def run_view_config(config_path=None): +def _require_config(config_path, console): + """Resolved config path, or None after reporting that it's missing.""" config_path = config_path or default_config_path() - console = Console() - if not os.path.isfile(config_path): console.print(f"[red]Config file not found: {config_path}[/red]") + return None + return config_path + + +def run_view_config(config_path=None): + console = Console() + config_path = _require_config(config_path, console) + if config_path is None: return with open(config_path) as f: @@ -288,19 +294,12 @@ def _navigate(screen, node, breadcrumb): def run_change_config(config_path=None): - config_path = config_path or default_config_path() console = Console() - - if not os.path.isfile(config_path): - console.print(f"[red]Config file not found: {config_path}[/red]") + config_path = _require_config(config_path, console) + if config_path is None: return - # Round-trip loader preserves comments, key order, quoting and ${ENV} refs. - yaml_rt = YAML() - yaml_rt.preserve_quotes = True - # Match the project's YAML style so edits don't reflow list indentation: - # block sequences indented under their key (` - item`). - yaml_rt.indent(mapping=2, sequence=4, offset=2) + yaml_rt = round_trip_yaml() with open(config_path) as f: data = yaml_rt.load(f) diff --git a/cli/canyonos/constants.py b/cli/canyonos/constants.py index d34316e..455e860 100644 --- a/cli/canyonos/constants.py +++ b/cli/canyonos/constants.py @@ -1,9 +1,60 @@ -"""Shared constants for the canyonos CLI.""" +"""Shared helpers for the canyonos CLI.""" import os +import yaml +from ruamel.yaml import YAML + +DEFAULT_API_PORT = 8080 + +# The workflow entrypoint is always exposed as POST /main with a {"query": ...} +# body, regardless of what the workflow function is called in the project. +WORKFLOW_ROUTE = "main" + def default_config_path(): """Global controller config for the current directory, preferring the .car artifact layout.""" car = os.path.join(".car", "config", "global_controller.yaml") return car if os.path.isfile(car) else os.path.join("config", "global_controller.yaml") + + +def workflow_api_port(config_path): + """Host port the workflow answers on, or None if there isn't one to read.""" + try: + with open(config_path) as f: + config = yaml.safe_load(f) or {} + except (OSError, yaml.YAMLError): + return None + + for agent in config.get("agents") or []: + if agent.get("type") == "workflow": + return agent.get("api_port", DEFAULT_API_PORT) + return None + + +def workspace_relative(config_path): + """`config_path` relative to the cwd, or None if it falls outside it. + + The container only ever receives a copy of the current directory, and it + resolves what it's given against /workspace -- so an absolute path silently + discards that prefix and a `../` one escapes it. Both then 404 naming a + path that exists on the host, which reads as a bug in the wrong place. + """ + # realpath on both sides: a symlinked project dir (or macOS's /tmp -> + # /private/tmp) otherwise makes an in-project absolute path look external. + relative = os.path.relpath(os.path.realpath(config_path), os.path.realpath(os.getcwd())) + if relative == ".." or relative.startswith(f"..{os.sep}"): + return None + return relative + + +def round_trip_yaml(): + """Loader that preserves comments, key order, quoting and ${ENV} refs. + + The indent settings match the project's YAML style, so edits don't reflow + list indentation: block sequences stay indented under their key. + """ + yaml_rt = YAML() + yaml_rt.preserve_quotes = True + yaml_rt.indent(mapping=2, sequence=4, offset=2) + return yaml_rt diff --git a/cli/canyonos/dashboard.compose.yml b/cli/canyonos/dashboard.compose.yml index db775aa..0689749 100644 --- a/cli/canyonos/dashboard.compose.yml +++ b/cli/canyonos/dashboard.compose.yml @@ -19,11 +19,14 @@ services: depends_on: db: condition: service_healthy - # Published so a GC container can POST OTLP spans to /v1/traces via - # host.docker.internal; that route also renames ventis' `project_id` - # attribute to the `canyon.project.id` every dashboard query filters on. + # Published on all interfaces (not just 127.0.0.1) so a GC container can + # actually reach this via host.docker.internal -- Docker's host-gateway + # route doesn't reach ports bound to loopback only, so a 127.0.0.1 bind + # here silently black-holed every OTLP span export. That route also + # renames ventis' `project_id` attribute to the `canyon.project.id` every + # dashboard query filters on. ports: - - "127.0.0.1:3000:3000" + - "3000:3000" environment: DATABASE_URL: postgresql://canyonos:canyonos@db:5432/canyonos JWT_SECRET: ${CANYONOS_JWT_SECRET} diff --git a/cli/canyonos/dashboard_stack.py b/cli/canyonos/dashboard_stack.py index 6abcfb1..c606357 100644 --- a/cli/canyonos/dashboard_stack.py +++ b/cli/canyonos/dashboard_stack.py @@ -19,11 +19,6 @@ from datetime import datetime, timezone from pathlib import Path from typing import Callable -from urllib.parse import urlsplit, urlunsplit - -import yaml - -from canyonos.constants import default_config_path COMPOSE_PROJECT = "canyonos-dashboard" STACK_VERSION = "v0.1.0-rc.2" @@ -32,7 +27,6 @@ HOST_GATEWAY = "host.docker.internal" REDIS_HOST = HOST_GATEWAY REDIS_PORT = "6379" -ENV_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") @dataclass(frozen=True) @@ -54,7 +48,6 @@ def __init__(self, phase: str, message: str, *, had_containers: bool | None = No @dataclass(frozen=True) class DashboardStack: - database_url: str | None state_dir: Path project_dir: Path web_port: int = 8080 @@ -73,9 +66,6 @@ def _state_dir() -> Path: def _compose_argv(stack: DashboardStack, manifest: Path) -> list[str]: - # Absolute, not "./.env": `canyonos serve` may cd into .car/ before - # running, so a cwd-relative path would miss the project root .env that - # `prepare()` actually writes to (stack.env_path). return [ "docker", "compose", @@ -88,25 +78,6 @@ def _compose_argv(stack: DashboardStack, manifest: Path) -> list[str]: ] -def _managed_database_url(database_url: str) -> tuple[str, str | None]: - parsed = urlsplit(database_url) - if parsed.hostname not in {"localhost", "127.0.0.1"}: - return database_url, None - - hostname = parsed.hostname - credentials = "" - if parsed.username is not None: - credentials = parsed.username - if parsed.password is not None: - credentials = f"{credentials}:{parsed.password}" - credentials = f"{credentials}@" - port = f":{parsed.port}" if parsed.port is not None else "" - rewritten = urlunsplit( - (parsed.scheme, f"{credentials}{HOST_GATEWAY}{port}", parsed.path, parsed.query, parsed.fragment) - ) - return rewritten, hostname - - def _existing_dashboard_port() -> int | None: """The host port an already-running dashboard `web` container owns, if any -- so re-running `canyonos serve` reconnects to the same stack instead of @@ -143,9 +114,9 @@ def _port_is_free(port: int) -> bool: def _find_web_port(start: int = 8080, max_attempts: int = 50) -> int: - """First free port at or after `start` -- same retry-on-conflict shape as - init.py's GC port selection, so an unrelated process/container squatting - on 8080 (e.g. a deployed Workflow's own api_port) doesn't hard-block serve. + """First free port at or after `start`, so an unrelated process or container + squatting on 8080 (e.g. a deployed Workflow's own api_port) doesn't + hard-block serve. """ for port in range(start, start + max_attempts): if _port_is_free(port): @@ -155,42 +126,7 @@ def _find_web_port(start: int = 8080, max_attempts: int = 50) -> int: ) -def _load_project_config(config_path: str) -> tuple[object, Path]: - project_root = Path(os.path.abspath(os.path.join(os.path.dirname(config_path), ".."))) - # `canyonos serve` cds into .car/ before calling here, so the naive - # parent-of-parent lands on .car itself -- go up one more level to reach - # the actual project root, where .env lives. - if project_root.name == ".car": - project_root = project_root.parent - dotenv_path = project_root / ".env" - if dotenv_path.is_file(): - with dotenv_path.open(encoding="utf-8") as dotenv: - for line in dotenv: - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - key = key.strip() - value = _env_value(value) - if key and key not in os.environ: - os.environ[key] = value - - with open(config_path, encoding="utf-8") as config_file: - config = yaml.safe_load(config_file) - return _expand_env_value(config), project_root - - -def _expand_env_value(value: object) -> object: - if isinstance(value, str): - return ENV_REFERENCE.sub(lambda match: os.environ.get(match.group(1), match.group(0)), value) - if isinstance(value, dict): - return {key: _expand_env_value(item) for key, item in value.items()} - if isinstance(value, list): - return [_expand_env_value(item) for item in value] - return value - - -def validate(config_path: str) -> DashboardStack: +def validate() -> DashboardStack: if shutil.which("docker") is None: raise PhaseFailure("validate", "docker is not on PATH") @@ -202,26 +138,10 @@ def validate(config_path: str) -> DashboardStack: except OSError: raise PhaseFailure("validate", "docker daemon or socket is unavailable") - try: - # Keep interpolation consistent with GlobalController._load_config in ventis/controller/global_controller.py. - config, project_root = _load_project_config(config_path) - except (OSError, yaml.YAMLError): - raise PhaseFailure("validate", f"config file is not readable: {config_path}") - - # database.url is optional -- the dashboard works without a database configured - # (e.g. OTLP-only setups); if present, it still needs to actually be usable. - database = config.get("database") if isinstance(config, dict) else None - database_url = database.get("url") if isinstance(database, dict) else None - if database_url is not None: - if not isinstance(database_url, str) or not database_url.strip(): - raise PhaseFailure("validate", "database.url must be a non-empty string") - unresolved = ENV_REFERENCE.search(database_url) - if unresolved: - name = unresolved.group(1) - raise PhaseFailure( - "validate", f"database.url needs ${{{name}}}, which is not set in the project .env" - ) - database_url = database_url.strip() + # The dashboard reads no project config -- it always runs against the + # bundled Postgres on this machine -- so the project root is just the cwd, + # the same assumption sync/clean/build already make. + project_root = Path.cwd() state_dir = _state_dir() try: @@ -235,7 +155,7 @@ def validate(config_path: str) -> DashboardStack: web_port = _existing_dashboard_port() or _find_web_port() - return DashboardStack(database_url, state_dir, project_root, web_port) + return DashboardStack(state_dir, project_root, web_port) def _env_value(value: str) -> str: @@ -319,10 +239,6 @@ def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: "CANYONOS_WEB_IMAGE": WEB_IMAGE, "CANYONOS_WEB_PORT": str(stack.web_port), } - rewritten_host = None - if stack.database_url is not None: - managed_database_url, rewritten_host = _managed_database_url(stack.database_url) - managed_env["CANYONOS_DATABASE_URL"] = managed_database_url _write_project_env(stack.env_path, managed_env) (stack.state_dir / "stack.json").write_text( json.dumps( @@ -338,20 +254,7 @@ def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: except (OSError, ValueError): raise PhaseFailure("prepare", "could not prepare the dashboard state directory") - message = "dashboard state prepared" - if rewritten_host: - message = ( - f"database host {rewritten_host} is reachable from the stack as host.docker.internal" - ) - return managed_env, message - - -def _redact_stack_text(text: str, stack: DashboardStack, managed_env: dict[str, str]) -> str: - secret = managed_env["CANYONOS_JWT_SECRET"] - redacted = redact_logs(text, secret) - if stack.database_url is not None: - redacted = redact_logs(redacted.replace(stack.database_url, "[redacted]"), secret) - return redacted + return managed_env, "dashboard state prepared" def _last_stderr_line(result: subprocess.CompletedProcess[str]) -> str | None: @@ -361,13 +264,12 @@ def _last_stderr_line(result: subprocess.CompletedProcess[str]) -> str | None: def _command_failure_message( message: str, result: subprocess.CompletedProcess[str], - stack: DashboardStack, managed_env: dict[str, str], ) -> str: detail = _last_stderr_line(result) if detail is None: return message - return f"{message}: {_redact_stack_text(detail, stack, managed_env)}" + return f"{message}: {redact_logs(detail, managed_env['CANYONOS_JWT_SECRET'])}" def pull( @@ -383,7 +285,7 @@ def pull( if result.returncode != 0: raise PhaseFailure( "pull", - _command_failure_message("docker compose pull failed", result, stack, managed_env), + _command_failure_message("docker compose pull failed", result, managed_env), had_containers=had_containers, ) return "dashboard images pulled" @@ -412,7 +314,7 @@ def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> if result.returncode != 0: raise PhaseFailure( "start", - _command_failure_message("docker compose up failed", result, stack, managed_env), + _command_failure_message("docker compose up failed", result, managed_env), had_containers=had_containers, ) return had_containers @@ -462,7 +364,7 @@ def _capture_failure_logs(stack: DashboardStack, manifest: Path, managed_env: di log_path = log_dir / f"serve-{timestamp}.log" _write_private_file( log_path, - _redact_stack_text(logs, stack, managed_env), + redact_logs(logs, managed_env["CANYONOS_JWT_SECRET"]), ) return log_path @@ -475,10 +377,8 @@ def _cleanup(stack: DashboardStack, manifest: Path) -> None: def run_dashboard( - config_path: str | None = None, phase_reporter: Callable[[str, str], None] | None = None, ) -> ServeResult: - config_path = config_path or default_config_path() def report(result: ServeResult) -> None: if phase_reporter is not None: phase_reporter(result.phase, result.message) @@ -489,7 +389,7 @@ def report(result: ServeResult) -> None: had_containers = False with ExitStack() as resources: try: - stack = validate(config_path) + stack = validate() report(ServeResult(True, "validate", "dashboard prerequisites validated")) managed_env, prepare_message = prepare(stack) diff --git a/cli/canyonos/deploy.py b/cli/canyonos/deploy.py index a2239a2..57c32ad 100644 --- a/cli/canyonos/deploy.py +++ b/cli/canyonos/deploy.py @@ -10,12 +10,20 @@ manual step. """ -import json import subprocess -import urllib.error -import urllib.request -from canyonos.constants import default_config_path +from rich.console import Console +from rich.panel import Panel +from rich.text import Text + +from canyonos.constants import ( + WORKFLOW_ROUTE, + default_config_path, + workflow_api_port, + workspace_relative, +) +from canyonos.gc import GCError, post_deploy +from canyonos.theme import GREEN, WHITE from canyonos.init import load_state, run_init from canyonos.serve import run_serve from canyonos.sync import run_sync @@ -27,7 +35,13 @@ def run_deploy(config_path=None, serve=True): - config_path = config_path or default_config_path() + # Left as None when unset: ventis resolves the artifact layout itself. + if config_path is not None: + config_path = workspace_relative(config_path) + if config_path is None: + print("Config must be inside the project directory being synced.") + return + run_init() # Copy the current project into the container before building/deploying. @@ -36,29 +50,53 @@ def run_deploy(config_path=None, serve=True): state = load_state() - url = f"http://127.0.0.1:{state['port']}/deploy" - body = json.dumps({"config_path": config_path}).encode() - req = urllib.request.Request( - url, data=body, headers={"Content-Type": "application/json"}, method="POST" - ) + # Read for display only -- ventis resolves the path it actually deploys. + api_port = workflow_api_port(config_path or default_config_path()) try: - with urllib.request.urlopen(req) as resp: - json.loads(resp.read()) - _stream_logs_and_autoserve(state["container_id"], serve=serve) - except urllib.error.HTTPError as e: - data = json.loads(e.read()) - print(f"Deploy failed: {data.get('error')}") - except urllib.error.URLError as e: - print(f"Could not reach Global Controller container: {e}") - - -def _stream_logs_and_autoserve(container_id, serve=True): - """Tail the GC container's logs (same as before), and -- unless disabled - via `serve=False` -- launch `canyonos serve` the moment they show the - workflow is up, so the dashboard is ready alongside it. Log tailing - continues afterwards exactly as before. + post_deploy(state["port"], config_path) + _stream_logs_and_autoserve(state["container_id"], api_port, serve=serve) + except GCError as e: + print(e) + + +def print_workflow_endpoint(console, api_port): + """The one thing you need after a deploy: where to send requests. + + Printed at the workflow-up marker and again on exit, because `deploy` keeps + tailing logs afterwards and would otherwise scroll it out of sight. + """ + if api_port is None: + return + + url = f"http://127.0.0.1:{api_port}/{WORKFLOW_ROUTE}" + body = Text.assemble( + ("POST ", "dim"), + (url, f"bold {GREEN}"), + ("\nbody ", "dim"), + ('{"query": "your question here"}', WHITE), + ("\npoll ", "dim"), + (f"http://127.0.0.1:{api_port}/status/", WHITE), + ) + console.print() + console.print( + Panel( + body, + title=f"[bold {GREEN}]Workflow is live[/]", + title_align="left", + border_style=GREEN, + padding=(1, 4), + ) + ) + console.print() + + +def _stream_logs_and_autoserve(container_id, api_port, serve=True): + """Tail the GC container's logs, and once they show the workflow is up, + print where to reach it -- plus, unless disabled via `serve=False`, launch + `canyonos serve`. Log tailing continues afterwards. """ + console = Console() process = subprocess.Popen( ["docker", "logs", "-f", container_id], stdout=subprocess.PIPE, @@ -67,20 +105,26 @@ def _stream_logs_and_autoserve(container_id, serve=True): bufsize=1, ) served = not serve + workflow_up = False try: for line in process.stdout: print(line, end="") - if not served and _WORKFLOW_UP_MARKER in line: - served = True - print("\nWorkflow is up -- starting the local dashboard (canyonos serve)...") - try: - run_serve() - except Exception as e: - print(f"Could not start the dashboard automatically: {e}") - print("Run `canyonos serve` manually to view it.") + if not workflow_up and _WORKFLOW_UP_MARKER in line: + workflow_up = True + print_workflow_endpoint(console, api_port) + if not served: + served = True + print("Starting the local dashboard (canyonos serve)...") + try: + run_serve() + except Exception as e: + print(f"Could not start the dashboard automatically: {e}") + print("Run `canyonos serve` manually to view it.") except KeyboardInterrupt: print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") print("To resubscribe to log stream run `canyonos logs`.") + if workflow_up: + print_workflow_endpoint(console, api_port) finally: if process.poll() is None: process.terminate() diff --git a/cli/canyonos/doctor.py b/cli/canyonos/doctor.py new file mode 100644 index 0000000..7dd9883 --- /dev/null +++ b/cli/canyonos/doctor.py @@ -0,0 +1,96 @@ +""" +Logic for `canyonos doctor`: a simple checklist of environment checks +(Docker installed/running, Compose available). Each check just reports +pass/fail plus a suggested fix -- nothing here attempts to auto-fix anything. +""" + +import shutil +import subprocess + +from canyonos.build import AGENTS +from canyonos.init import docker_start_command + + +def _docker_installed(): + return shutil.which("docker") is not None + + +def _docker_daemon_running(): + result = subprocess.run(["docker", "info"], capture_output=True) + return result.returncode == 0 + + +def _compose_available(): + result = subprocess.run(["docker", "compose", "version"], capture_output=True) + return result.returncode == 0 + + +def _git_available(): + return shutil.which("git") is not None + + +def _docker_daemon_fix(): + """Names the command for the active docker context, since `canyonos deploy` + would run exactly that itself.""" + command = docker_start_command() + if command: + return f"run `{' '.join(command)}` -- or just run `canyonos deploy`, which starts it for you" + return "start your Docker runtime (on Linux: `sudo systemctl start docker`)" + + +def _coding_agent_available(): + return any(shutil.which(spec["cli"]) for spec in AGENTS.values()) + + +def _checks(): + """Built fresh on each call (not a module-level constant) so tests can + patch the individual `_check_*` functions by name and have it take effect. + """ + return [ + ( + "Docker installed", + _docker_installed, + "install Docker: https://docs.docker.com/get-docker/", + ), + ( + "Docker daemon running", + _docker_daemon_running, + _docker_daemon_fix(), + ), + ( + "Docker Compose available", + _compose_available, + "update Docker to a version that includes Compose v2 (needed for `canyonos serve`)", + ), + ( + "git available", + _git_available, + "install git (`canyonos build` fetches the porting skill with it; " + "without git it falls back to a full-repo tarball download)", + ), + ( + "Coding agent available", + _coding_agent_available, + "install one of " + + " or ".join(spec["label"] for spec in AGENTS.values()) + + " (`canyonos build` runs the port through it)", + ), + ] + + +def run_doctor(): + """Run every check, print a pass/fail checklist, and return True iff all passed.""" + all_ok = True + for label, check, fix in _checks(): + try: + passed = bool(check()) + except OSError as e: + passed = False + fix = f"{fix} (error: {e})" + + print(f"{'✓' if passed else '✗'} {label}") + if not passed: + print(f" -> {fix}") + all_ok = False + + return all_ok diff --git a/cli/canyonos/gc.py b/cli/canyonos/gc.py new file mode 100644 index 0000000..b5af7eb --- /dev/null +++ b/cli/canyonos/gc.py @@ -0,0 +1,83 @@ +""" +Shared request helpers for the Global Controller container, so the commands +that talk to it don't each restate the same routes, payloads and failure modes. +""" + +import json +import urllib.error +import urllib.request + +from canyonos.init import load_state + +_DEPLOY_CONFLICT = "Run `canyonos stop` to stop the running deploy first." + + +class GCError(Exception): + """A failed Global Controller request, carrying a message fit to print.""" + + def __init__(self, message, code=None): + super().__init__(message) + self.code = code + + +def _error_detail(e): + """The server's `error` field, falling back to the raw body when it isn't JSON.""" + body = e.read().decode(errors="replace").strip() + try: + return json.loads(body).get("error", body) + except ValueError: + return body or f"HTTP {e.code}" + + +def _request(url, action, data=None, method="GET"): + headers = {"Content-Type": "application/json"} if data is not None else {} + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as e: + raise GCError(f"{action} failed: {_error_detail(e)}", code=e.code) from None + except urllib.error.URLError as e: + raise GCError(f"Could not reach Global Controller container: {e.reason}") from None + + +def require_state(): + """Recorded container state, or None after reporting that there is none.""" + try: + return load_state() + except FileNotFoundError: + print("No Global Controller container is running. Run `canyonos deploy` first.") + return None + + +def post_deploy(port, config_path=None): + """Start a deploy inside the container. Raises GCError on failure. + + Omitting config_path lets ventis resolve it against the synced workspace. + """ + body = json.dumps({"config_path": config_path} if config_path else {}).encode() + try: + return _request(f"http://127.0.0.1:{port}/deploy", "Deploy", data=body, method="POST") + except GCError as e: + if e.code == 409: + raise GCError(f"{e}\n{_DEPLOY_CONFLICT}", code=409) from None + raise + + +def post_clean(port): + """Tear down the running deploy: SIGTERMs the in-container `ventis deploy` + process, whose handler calls GlobalController.stop() and blocks until it + returns. This is what actually removes the local controller and Redis + containers a deploy spawned via docker-outside-of-docker. + """ + return _request(f"http://127.0.0.1:{port}/clean", "Stop", method="POST") + + +def deploy_status(port): + """Parsed /status payload, or None if the container is unreachable.""" + url = f"http://127.0.0.1:{port}/status" + try: + with urllib.request.urlopen(url, timeout=5) as resp: + return json.loads(resp.read()) + except OSError: + return None diff --git a/cli/canyonos/init.py b/cli/canyonos/init.py index 3d9cee9..69e8fbb 100644 --- a/cli/canyonos/init.py +++ b/cli/canyonos/init.py @@ -7,7 +7,10 @@ import json import os +import shutil import subprocess +import sys +import time import urllib.error import urllib.request @@ -23,22 +26,95 @@ GC_IMAGE = "saakeths/canyonos:latest" GC_CONTAINER_PORT = 8000 -# Named docker volume mounted at /workspace inside the container. Unlike a bind -# mount, this lives in the container's docker volume (not the host filesystem): -# it persists across `canyonos quit` (docker rm leaves named volumes intact) and -# is unaffected by host-side changes. Files are copied in via `canyonos sync` -# (docker cp), not mounted live. +# Named docker volume mounted at /workspace inside the container. Files are +# copied in via `canyonos sync` (docker cp), not mounted live, so host-side +# edits don't reach a running build. `canyonos quit` removes the volume, and +# since every deploy quits any previous controller first, each deploy starts +# from an empty workspace. GC_WORKSPACE_VOLUME = "canyonos-workspace" GC_WORKSPACE_PATH = "/workspace" STATE_DIR = os.path.expanduser("~/.canyonos") STATE_PATH = os.path.join(STATE_DIR, "state.json") +# How to start the daemon behind each docker context, as (CLI command, macOS +# app). Keyed off the *active context* rather than which app is installed: with +# both Docker Desktop and OrbStack present, guessing by app bundle starts the +# wrong daemon and then waits out the timeout against a socket nothing is +# listening on. +DOCKER_RUNTIMES = { + "orbstack": (["orb", "start"], "OrbStack"), + "colima": (["colima", "start"], None), + "desktop-linux": (None, "Docker"), + "default": (None, "Docker"), +} +DOCKER_START_TIMEOUT = 60 + + +def docker_running(): + try: + return subprocess.run(["docker", "info"], capture_output=True).returncode == 0 + except OSError: + return False + + +def docker_start_command(): + """The command that starts the daemon for the active context, or None.""" + try: + result = subprocess.run( + ["docker", "context", "show"], capture_output=True, text=True + ) + except OSError: + return None + + context = result.stdout.strip() if result.returncode == 0 else "default" + command, app = DOCKER_RUNTIMES.get(context, (None, "Docker")) + if command and shutil.which(command[0]): + return command + if app and sys.platform == "darwin" and os.path.isdir(f"/Applications/{app}.app"): + return ["open", "-a", app] + return None + + +def ensure_docker_running(console, timeout=DOCKER_START_TIMEOUT): + if docker_running(): + return + + command = docker_start_command() + if command is None: + # Linux/systemd wants root here; escalating on the user's behalf is not + # this CLI's call to make. + raise RuntimeError( + "Docker isn't running, and there's no way to start it for the current " + "docker context. Start it (on Linux: `sudo systemctl start docker`) and re-run." + ) + + console.print(f"Docker isn't running -- starting it with `{' '.join(command)}`...") + subprocess.run(command, capture_output=True) + + deadline = time.time() + timeout + with console.status("Waiting for the Docker daemon..."): + while time.time() < deadline: + if docker_running(): + console.print("Docker is running.") + return + time.sleep(1) + + raise RuntimeError( + f"Docker did not become ready within {timeout}s. Start it manually and re-run." + ) + def pull_image(image=GC_IMAGE): # Capture output so the rich status spinner isn't clobbered by docker's own - # layer-progress printing. - subprocess.run(["docker", "pull", image], check=True, capture_output=True) + # layer-progress printing -- but surface it on failure (auth, network, + # rate-limit, missing arch, etc. all otherwise look like the same opaque + # "exit status 1"). + result = subprocess.run(["docker", "pull", image], capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError( + f"docker pull {image} failed: {result.stderr.strip() or result.stdout.strip()}" + ) def _port_reachable(port, attempts=10, delay=0.5): @@ -49,8 +125,6 @@ def _port_reachable(port, attempts=10, delay=0.5): trigger it), which looks fine at the Docker level but resets every real connection. Confirm the container is actually reachable before trusting it. """ - import time - url = f"http://127.0.0.1:{port}/status" for _ in range(attempts): try: @@ -113,6 +187,19 @@ def load_state(): return json.load(f) +def quit_existing(): + """Tear down a previously started Global Controller, if state records one. + + Without this each run starts another container on the next free port and + orphans the last one, which then can't be reached through state.json. + """ + # Deferred: quit.py imports from this module, so a top-level import cycles. + from canyonos.quit import run_quit + + if os.path.isfile(STATE_PATH): + run_quit() + + def run_init(): console = Console() banner = figlet_format("CANYON OS", font="ansi_shadow", width=200) @@ -120,6 +207,9 @@ def run_init(): for line, color in zip(banner.splitlines(), GRADIENT): console.print(line, style=color) + # Before quit_existing(), which shells out to docker itself. + ensure_docker_running(console) + quit_existing() with console.status("Pulling Global Controller image..."): pull_image() diff --git a/cli/canyonos/integrate.py b/cli/canyonos/integrate.py deleted file mode 100644 index 23e61b1..0000000 --- a/cli/canyonos/integrate.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -Logic for `canyonos integrate`: install the CanyonOS skill on a coding agent, -then launch that agent with a prompt to apply it to the current project. -""" - -import os -import shutil -import subprocess - -from rich.console import Console - -from utils.tui import select_menu - -# Points at the skill's folder, so SKILL.md and references/ both come along. -SKILL_SOURCE_URL = "https://github.com/CanyonCodeCoreAI/canyoncodecore/tree/nickhuo/porting-skill-car-layout/.claude/skills/porting-to-canyonos" - -# The porting skill emits no otel config; without this the dashboard stays empty. -OTEL_BLOCK = """otel: - destinations: - - name: local - protocol: http - endpoint: http://host.docker.internal:3000/v1/traces - headers: {}""" - -INTEGRATE_PROMPT = ( - "Use the CanyonOS porting-to-canyonos-core skill to convert the codebase in this directory to a canyonos-compatable format. No changes should be made to the current files, but all modifications should be put into a new .car folder." - "\n\nFinally, add the following block verbatim to the generated config/global_controller.yaml," - " at the top level as a sibling of `agents:`. Copy it exactly -- `protocol` must be http, and" - " the endpoint must keep the /v1/traces path:\n\n" + OTEL_BLOCK -) - -AGENTS = { - "claude": { - "label": "Claude Code", - "cli": "claude", - # Claude Code auto-loads project-local skills from here. - "skill_dir": ".claude/skills/porting-to-canyonos-core", - }, - "codex": { - "label": "Codex", - "cli": "codex", - # Codex only auto-loads skills from the user's home directory, not per-project. - "skill_dir": os.path.expanduser("~/.codex/skills/porting-to-canyonos-core"), - }, -} - - -def prompt_agent(): - options = [(key, spec["label"]) for key, spec in AGENTS.items()] - return select_menu(options, title="Which coding agent do you want to integrate with?") - - -def install_skill(agent): - spec = AGENTS[agent] - # -f overwrites an existing skill dir; without it gitpick exits 1 when the - # target already exists and is non-empty (e.g. re-running `integrate`). - subprocess.run( - ["npx", "-y", "gitpick", "-f", SKILL_SOURCE_URL, spec["skill_dir"]], - check=True, - ) - - -def launch_agent(agent, prompt): - spec = AGENTS[agent] - if not shutil.which(spec["cli"]): - print(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") - return - subprocess.run([spec["cli"], prompt], check=True) - - -def run_integrate(): - console = Console() - agent = prompt_agent() - if agent is None: - console.print("Cancelled.") - return - - console.print(f"Installing CanyonOS skill for {AGENTS[agent]['label']}...") - install_skill(agent) - - console.print(f"Launching {AGENTS[agent]['label']}...") - launch_agent(agent, INTEGRATE_PROMPT) diff --git a/cli/canyonos/logs.py b/cli/canyonos/logs.py index 9b2b1d8..9839134 100644 --- a/cli/canyonos/logs.py +++ b/cli/canyonos/logs.py @@ -2,32 +2,22 @@ Logic for `canyonos logs`: re-subscribe to the running deploy's log stream. """ -import json import subprocess -import urllib.error -import urllib.request -from canyonos.init import load_state +from canyonos.gc import deploy_status, require_state def run_logs(): - try: - state = load_state() - except FileNotFoundError: - print("No Global Controller container is running. Run `canyonos init` first.") + state = require_state() + if state is None: return - url = f"http://127.0.0.1:{state['port']}/status" - req = urllib.request.Request(url, method="GET") - - try: - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) - except urllib.error.URLError as e: - print(f"Could not reach Global Controller container: {e}") + status = deploy_status(state["port"]) + if status is None: + print("Could not reach Global Controller container.") return - if not data.get("running"): + if not status.get("running"): print("No deploy running, run `canyonos deploy` to deploy project.") return diff --git a/cli/canyonos/quit.py b/cli/canyonos/quit.py index 15aff2c..bb2e11d 100644 --- a/cli/canyonos/quit.py +++ b/cli/canyonos/quit.py @@ -10,8 +10,8 @@ from rich.console import Console -from canyonos.init import GC_WORKSPACE_VOLUME, STATE_PATH, load_state -from canyonos.stop import _post_clean +from canyonos.gc import GCError, post_clean, require_state +from canyonos.init import GC_WORKSPACE_VOLUME, STATE_PATH def _container_exists(container_id): @@ -22,10 +22,8 @@ def _container_exists(container_id): def run_quit(): - try: - state = load_state() - except FileNotFoundError: - print("No Global Controller container is running.") + state = require_state() + if state is None: return container_id = state["container_id"] @@ -36,11 +34,9 @@ def run_quit(): # too. Removing the GC container itself doesn't touch them -- they're # sibling containers on the host, not nested inside it. try: - _post_clean(state["port"]) - except OSError: - # Covers urllib.error.HTTPError/URLError (both subclass OSError) - # plus raw connection errors -- nothing was running, or the GC is - # already unreachable/gone. + post_clean(state["port"]) + except GCError: + # Nothing was running, or the GC is already unreachable/gone. pass # state.json can go stale (daemon restarted, container removed by diff --git a/cli/canyonos/serve.py b/cli/canyonos/serve.py index ddfd7d6..9515699 100644 --- a/cli/canyonos/serve.py +++ b/cli/canyonos/serve.py @@ -3,11 +3,11 @@ from .dashboard_stack import run_dashboard -def run_serve(config_path: str | None = None) -> int: +def run_serve() -> int: def report(phase: str, message: str) -> None: print(f"[serve] {phase}: {message}") - result = run_dashboard(config_path, report) + result = run_dashboard(report) if result.ok: print(f"Dashboard: {result.url}") return 0 diff --git a/cli/canyonos/stop.py b/cli/canyonos/stop.py index 2f6ad40..a3f96f3 100644 --- a/cli/canyonos/stop.py +++ b/cli/canyonos/stop.py @@ -3,45 +3,20 @@ Controller container (SIGTERM, same teardown as Ctrl+C would trigger). """ -import json -import urllib.error -import urllib.request - from rich.console import Console -from canyonos.init import load_state - - -def _post_clean(port): - """POST /clean to the Global Controller container. - - This is what actually tears down the local controller and Redis - containers a deploy spawned via docker-outside-of-docker: it sends - SIGTERM to the in-container `ventis deploy` process, whose handler calls - `GlobalController.stop()` and blocks until it returns. Shared with - `canyonos quit`, which needs the same teardown before removing the GC - container itself. - """ - url = f"http://127.0.0.1:{port}/clean" - req = urllib.request.Request(url, method="POST") - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read()) +from canyonos.gc import GCError, post_clean, require_state def run_stop(): - try: - state = load_state() - except FileNotFoundError: - print("No Global Controller container is running. Run `canyonos init` first.") + state = require_state() + if state is None: return console = Console() try: with console.status("Stopping deploy..."): - _post_clean(state["port"]) + post_clean(state["port"]) print("Deploy stopped.") - except urllib.error.HTTPError as e: - data = json.loads(e.read()) - print(f"Stop failed: {data.get('error')}") - except urllib.error.URLError as e: - print(f"Could not reach Global Controller container: {e}") + except GCError as e: + print(e) diff --git a/cli/canyonos/sync.py b/cli/canyonos/sync.py index f350a1c..20ac3c1 100644 --- a/cli/canyonos/sync.py +++ b/cli/canyonos/sync.py @@ -3,24 +3,24 @@ Controller container's /workspace volume via `docker cp`. Files live inside the container's named volume (see `init.py`), not on a live -bind mount -- so they persist across `canyonos quit` and survive host-side -changes. `docker cp` is additive: it overwrites/adds files but never deletes, -so build outputs generated inside the container (stubs/, grpc_stubs/, -docker_container/) survive a re-sync of the host source. +bind mount, so host-side edits don't reach a running build. `docker cp` is +additive -- it overwrites and adds but never deletes -- so a standalone +re-sync leaves behind anything removed from the host since the last one. +That can't accumulate across deploys: `canyonos deploy` quits any previous +controller first, which removes the volume. """ import os import subprocess -from canyonos.init import GC_WORKSPACE_PATH, load_state +from canyonos.gc import require_state +from canyonos.init import GC_WORKSPACE_PATH def run_sync(): """Copy the current directory into the container. Returns True on success.""" - try: - state = load_state() - except FileNotFoundError: - print("No Global Controller container is running. Run `canyonos init` first.") + state = require_state() + if state is None: return False container_id = state["container_id"] diff --git a/cli/canyonos/test.py b/cli/canyonos/test.py new file mode 100644 index 0000000..468527a --- /dev/null +++ b/cli/canyonos/test.py @@ -0,0 +1,173 @@ +""" +Logic for `canyonos test`: smoke-test a project end to end on this machine. + +Every agent's `provider` is rewritten to `local` for the duration of the run +(the original file is restored verbatim afterwards), the project is deployed +into the Global Controller container, one query is sent to the workflow's +`/main` endpoint, and its result -- or the error that came back -- is printed. +""" + +import json +import os +import time +import urllib.error +import urllib.request + +from rich.console import Console + +from canyonos.constants import ( + WORKFLOW_ROUTE, + default_config_path, + round_trip_yaml, + workflow_api_port, + workspace_relative, +) +from canyonos.gc import GCError, deploy_status, post_deploy +from canyonos.init import load_state, quit_existing, run_init +from canyonos.sync import run_sync + +DEFAULT_QUERY = "hello" +# Generous: the first deploy of a project builds every agent image from scratch. +READY_TIMEOUT = 900 +REQUEST_TIMEOUT = 600 +POLL_INTERVAL = 2 + + + +def _force_local_providers(config_path): + """Set every agent's provider to `local`. Returns the original file text.""" + with open(config_path) as f: + original = f.read() + + yaml_rt = round_trip_yaml() + data = yaml_rt.load(original) + + for agent in data.get("agents") or []: + agent["provider"] = "local" + + with open(config_path, "w") as f: + yaml_rt.dump(data, f) + + return original + + +def _workflow_ready(api_port): + """True once the workflow's REST API answers at all. + + Any HTTP response counts -- /status/ 404s, which still proves + the server is up and listening. + """ + url = f"http://127.0.0.1:{api_port}/status/canyonos-test-probe" + try: + urllib.request.urlopen(url, timeout=2) + return True + except urllib.error.HTTPError: + return True + except OSError: + return False + + +def _wait_for_workflow(gc_port, api_port, console): + deadline = time.time() + READY_TIMEOUT + with console.status("Building and starting containers..."): + while time.time() < deadline: + if _workflow_ready(api_port): + return True + if not (deploy_status(gc_port) or {}).get("running", False): + return False + time.sleep(POLL_INTERVAL) + print(f"Timed out after {READY_TIMEOUT}s waiting for the workflow to come up.") + return False + + +def _send_query(api_port, query): + url = f"http://127.0.0.1:{api_port}/{WORKFLOW_ROUTE}" + body = json.dumps({"query": query}).encode() + req = urllib.request.Request( + url, data=body, headers={"Content-Type": "application/json"}, method="POST" + ) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read())["request_id"] + + +def _await_result(api_port, request_id, console): + url = f"http://127.0.0.1:{api_port}/status/{request_id}" + deadline = time.time() + REQUEST_TIMEOUT + with console.status("Running query..."): + while time.time() < deadline: + try: + with urllib.request.urlopen(url, timeout=10) as resp: + data = json.loads(resp.read()) + if data.get("status") in ("done", "error"): + return data + except OSError: + # A blip while the workflow is busy; keep polling until the deadline. + pass + time.sleep(POLL_INTERVAL) + return {"status": "timeout"} + + +def run_test(config_path=None, query=None): + console = Console() + config_path = config_path or default_config_path() + query = query or DEFAULT_QUERY + + config_path = workspace_relative(config_path) + if config_path is None: + print("Config must be inside the project directory being synced.") + return 1 + + if not os.path.isfile(config_path): + print(f"Config file not found: {config_path}. Run `canyonos build` first.") + return 1 + + api_port = workflow_api_port(config_path) + if api_port is None: + print(f"No agent with `type: workflow` in {config_path}; nothing to test.") + return 1 + + print(f"Testing {config_path} locally (query: {query!r})") + original_config = _force_local_providers(config_path) + + try: + run_init() + if not run_sync(): + return 1 + + state = load_state() + try: + post_deploy(state["port"], config_path) + except GCError as e: + print(e) + return 1 + + if not _wait_for_workflow(state["port"], api_port, console): + print("The deploy did not come up. Run `canyonos logs` to see why.") + return 1 + + try: + request_id = _send_query(api_port, query) + except OSError as e: + print(f"Could not reach the workflow on port {api_port}: {e}") + return 1 + result = _await_result(api_port, request_id, console) + except KeyboardInterrupt: + print("\nTest cancelled.") + return 1 + finally: + with open(config_path, "w") as f: + f.write(original_config) + # A smoke test leaves nothing behind: run_init() started this container. + quit_existing() + + status = result.get("status") + if status == "done": + print("Test passed.") + print(json.dumps(result.get("result"), indent=2)) + return 0 + + if status == "error": + print(f"Test failed: {result.get('error')}") + else: + print(f"Test failed: workflow did not finish within {REQUEST_TIMEOUT}s.") + return 1 diff --git a/cli/cli.py b/cli/cli.py index 4c78043..770741f 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -1,42 +1,33 @@ """ -Most of the commands will be executed by code in the canyonos container. -Anything executing in this CLI pertains to file/folder modification +Almost all commands will be executing on the canyonos container spawned by deploy +Commands like doctor, version, and new_app will not though """ import argparse +import importlib.metadata import sys from canyonos.clean import run_clean from canyonos.constants import default_config_path from canyonos.config import run_config from canyonos.deploy import run_deploy -from canyonos.integrate import run_integrate +from canyonos.build import run_build +from canyonos.doctor import run_doctor from canyonos.logs import run_logs from canyonos.new_app import run_new_app from canyonos.quit import run_quit from canyonos.serve import run_serve from canyonos.stop import run_stop -from canyonos.sync import run_sync - -try: - from rich.console import Console - from rich.panel import Panel - from rich.text import Text - from rich.table import Table - RICH_AVAILABLE = True -except ImportError: - RICH_AVAILABLE = False - -def cmd_connect(args): - pass +from canyonos.test import DEFAULT_QUERY, run_test +from utils.help_screen import DESCRIPTIONS, print_custom_help def cmd_quit(args): run_quit() def cmd_new_app(args): + # Note, not tested much, keeping this in the back burner for now while we flesh out the main path run_new_app() -# Executed in canyonos: syncs files, then builds + deploys def cmd_deploy(args): run_deploy(args.config, serve=args.serve) @@ -49,111 +40,24 @@ def cmd_stop(args): def cmd_logs(args): run_logs() -def cmd_sync(args): - run_sync() - def cmd_config(args): run_config() -def cmd_integrate(args): - run_integrate() +def cmd_build(args): + run_build() def cmd_doctor(args): - pass + sys.exit(0 if run_doctor() else 1) def cmd_serve(args): - sys.exit(run_serve(args.config)) + sys.exit(run_serve()) -# Executed in canyonos def cmd_test(args): - pass - -# Executed in canyonos -def cmd_mega_build(args): - pass + sys.exit(run_test(args.config, query=args.query)) def cmd_version(args): - pass - - -def print_custom_help(): - """Print a custom, visually appealing help screen.""" - if RICH_AVAILABLE: - console = Console() - - # Header - title = Text("CanyonOS CLI", style="bold cyan") - subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease\n", style="dim") - - console.print(Panel(title + subtitle, border_style="cyan", padding=(1, 2))) - - # Core commands - console.print("\n[bold yellow]Core Commands[/bold yellow]") - core_table = Table(show_header=False, border_style="dim", padding=(0, 2)) - core_table.add_column(style="cyan", width=20) - core_table.add_column(style="white") - core_table.add_row("integrate", "Sync source files to .car/app/") - core_table.add_row("deploy", "Build and deploy agents to configured hosts") - core_table.add_row("config", "Configure project settings") - console.print(core_table) - - # Utils commands - console.print("\n[bold yellow]Utils[/bold yellow]") - utils_table = Table(show_header=False, border_style="dim", padding=(0, 2)) - utils_table.add_column(style="cyan", width=20) - utils_table.add_column(style="white") - utils_table.add_row("new-app", "Create a new CanyonOS project") - utils_table.add_row("serve", "Start local CanyonOS dashboard") - utils_table.add_row("sync", "Sync files with container") - utils_table.add_row("stop", "Stop running containers") - utils_table.add_row("clean", "Remove generated files") - utils_table.add_row("logs", "View container logs") - utils_table.add_row("doctor", "Check system health") - utils_table.add_row("connect", "Connect to remote host") - utils_table.add_row("quit", "Shut down CanyonOS services") - console.print(utils_table) - - # Quick start - console.print("\n[bold green]Quick Start:[/bold green]") - console.print(" [dim]1.[/dim] canyonos new-app [cyan]my-app[/cyan]") - console.print(" [dim]2.[/dim] cd [cyan]my-app[/cyan]") - console.print(" [dim]3.[/dim] canyonos integrate") - console.print(" [dim]4.[/dim] canyonos deploy") - console.print(" [dim]5.[/dim] canyonos serve\n") - - console.print("[dim]For command-specific help: [cyan]canyonos --help[/cyan][/dim]\n") - else: - # Fallback to simple text if rich is not available - print("\n" + "="*60) - print(" " * 20 + "CanyonOS CLI") - print(" " * 10 + "Build, deploy, and manage agentic workflows") - print("="*60 + "\n") - - print("CORE COMMANDS:") - print(" integrate Sync source files to .car/app/") - print(" deploy Build and deploy agents to configured hosts") - print(" config Configure project settings\n") - - print("UTILS:") - print(" new-app Create a new CanyonOS project") - print(" serve Start local CanyonOS dashboard") - print(" sync Sync files with container") - print(" stop Stop running containers") - print(" clean Remove generated files") - print(" logs View container logs") - print(" doctor Check system health") - print(" connect Connect to remote host") - print(" quit Shut down CanyonOS services\n") - - print("QUICK START:") - print(" 1. canyonos new-app my-app") - print(" 2. cd my-app") - print(" 3. canyonos integrate") - print(" 4. canyonos deploy") - print(" 5. canyonos serve\n") - - print("For command-specific help: canyonos --help\n") + print(f"canyonos {importlib.metadata.version('canyonos')}") def _parse_bool(value): @@ -164,18 +68,30 @@ def _parse_bool(value): raise argparse.ArgumentTypeError(f"expected true/false, got: {value!r}") +class _RootParser(argparse.ArgumentParser): + """Routes the top-level -h/--help through the custom help screen.""" + + def print_help(self, file=None): + print_custom_help() + + def main(): - parser = argparse.ArgumentParser(prog="canyonos") - subparsers = parser.add_subparsers(dest="command") + parser = _RootParser(prog="canyonos") + # Subparsers keep the stock argparse help, so `canyonos -h` still + # describes that command instead of reprinting the top-level screen. + subparsers = parser.add_subparsers(dest="command", parser_class=argparse.ArgumentParser) config_default = default_config_path() - subparsers.add_parser("new-app").set_defaults(func=cmd_new_app) - deploy = subparsers.add_parser("deploy") + def add(name): + # A KeyError here means the command has no entry on the help screen. + return subparsers.add_parser(name, help=DESCRIPTIONS[name]) + + add("new-app").set_defaults(func=cmd_new_app) + deploy = add("deploy") deploy.add_argument( "-c", "--config", - default=config_default, - help=f"Path to global controller config (default: {config_default})", + help="Path to global controller config (default: resolved by ventis inside the container)", ) deploy.add_argument( "--serve", @@ -185,32 +101,42 @@ def main(): help="Automatically launch the local dashboard (canyonos serve) once the workflow is up (default: true)", ) deploy.set_defaults(func=cmd_deploy) - subparsers.add_parser("clean").set_defaults(func=cmd_clean) - subparsers.add_parser("stop").set_defaults(func=cmd_stop) - subparsers.add_parser("logs").set_defaults(func=cmd_logs) - subparsers.add_parser("quit").set_defaults(func=cmd_quit) - subparsers.add_parser("connect").set_defaults(func=cmd_connect) - subparsers.add_parser("sync").set_defaults(func=cmd_sync) - subparsers.add_parser("config").set_defaults(func=cmd_config) - subparsers.add_parser("integrate").set_defaults(func=cmd_integrate) - subparsers.add_parser("doctor").set_defaults(func=cmd_doctor) - serve = subparsers.add_parser("serve") - serve.add_argument( + add("clean").set_defaults(func=cmd_clean) + add("stop").set_defaults(func=cmd_stop) + add("logs").set_defaults(func=cmd_logs) + add("quit").set_defaults(func=cmd_quit) + add("config").set_defaults(func=cmd_config) + add("build").set_defaults(func=cmd_build) + add("doctor").set_defaults(func=cmd_doctor) + add("version").set_defaults(func=cmd_version) + add("serve").set_defaults(func=cmd_serve) + test = add("test") + test.add_argument( "-c", "--config", default=config_default, help=f"Path to global controller config (default: {config_default})", ) - serve.set_defaults(func=cmd_serve) - subparsers.add_parser("test").set_defaults(func=cmd_test) - subparsers.add_parser("mega-build").set_defaults(func=cmd_mega_build) + test.add_argument( + "-q", + "--query", + default=DEFAULT_QUERY, + help=f"Query sent to the workflow (default: {DEFAULT_QUERY!r})", + ) + test.set_defaults(func=cmd_test) args = parser.parse_args() if not getattr(args, "command", None): - print_custom_help() + parser.print_help() return - args.func(args) + try: + args.func(args) + except RuntimeError as e: + # Docker unreachable, image pull failed, no free port -- all already + # carry a readable message, so print it rather than a traceback. + print(e) + sys.exit(1) if __name__ == "__main__": diff --git a/cli/utils/help_screen.py b/cli/utils/help_screen.py new file mode 100644 index 0000000..94b94a0 --- /dev/null +++ b/cli/utils/help_screen.py @@ -0,0 +1,60 @@ +"""Custom help screen for the canyonos CLI.""" + +from rich.console import Console +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +# The single source of truth for command descriptions: cli.py registers every +# subparser through this table, so a command can't be added to one and missed +# in the other. +CORE_COMMANDS = ( + ("build", "Build a compatable workflow with an agent"), + ("deploy", "Deploy agents"), + ("config", "Configure project settings"), +) + +UTIL_COMMANDS = ( + ("clean", "Remove the generated .car folder from build"), + ("doctor", "See if all required tools are up"), + ("logs", "View canyonos logs"), + ("new-app", "Create a barebones CanyonOS project"), + ("quit", "Shut down CanyonOS services"), + ("serve", "Start local CanyonOS dashboard"), + ("stop", "Stop running containers"), + ("test", "Run the deployed workflow locally with a test query"), + ("version", "Print canyonos version"), +) + +DESCRIPTIONS = dict(CORE_COMMANDS + UTIL_COMMANDS) + + +def _command_table(commands): + table = Table(show_header=False, border_style="dim", padding=(0, 2)) + table.add_column(style="cyan", width=20) + table.add_column(style="white") + for name, description in commands: + table.add_row(name, description) + return table + + +def print_custom_help(): + """Print a custom, visually appealing help screen.""" + console = Console() + + title = Text("CanyonOS CLI", style="bold cyan") + subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease\n", style="dim") + console.print(Panel(title + subtitle, border_style="cyan", padding=(1, 2))) + + console.print("\n[bold yellow]Core Commands[/bold yellow]") + console.print(_command_table(CORE_COMMANDS)) + + console.print("\n[bold yellow]Utils[/bold yellow]") + console.print(_command_table(UTIL_COMMANDS)) + + console.print("\n[bold green]Quick Start:[/bold green]") + console.print(" [dim]1.[/dim] cd [cyan]into-your-workflow-root-dir[/cyan]") + console.print(" [dim]2.[/dim] canyonos build (opens coding agent)") + console.print(" [dim]3.[/dim] canyonos deploy (UI automatically starts)") + + console.print("[dim]For command-specific help: [cyan]canyonos --help[/cyan][/dim]\n") diff --git a/cli/utils/tui.py b/cli/utils/tui.py index 3fcdad5..2f0f15d 100644 --- a/cli/utils/tui.py +++ b/cli/utils/tui.py @@ -53,8 +53,6 @@ def select_menu(options, title, deletable=False, quittable=False): `QUIT_ACTION` -- distinct from None -- so the caller can unwind an entire nested session rather than just this one menu. """ - if len(options) == 1: - return options[0][0] if not options or not sys.stdin.isatty(): return None diff --git a/cli/tests/test_dashboard_stack.py b/tests/test_dashboard_stack.py similarity index 100% rename from cli/tests/test_dashboard_stack.py rename to tests/test_dashboard_stack.py From 9447e348a511466397194fda3265f875dbe895f6 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 23:13:04 -0700 Subject: [PATCH 29/31] cleanup --- README.md | 4 +- cli/canyonos/build.py | 71 +-- cli/canyonos/clean.py | 10 +- cli/canyonos/config.py | 41 +- cli/canyonos/dashboard_stack.py | 78 ++-- cli/canyonos/deploy.py | 401 ++++++++++++++--- cli/canyonos/doctor.py | 39 +- cli/canyonos/gc.py | 16 +- cli/canyonos/init.py | 28 +- cli/canyonos/logs.py | 8 +- cli/canyonos/new_app.py | 6 +- cli/canyonos/quit.py | 10 +- cli/canyonos/serve.py | 39 +- cli/canyonos/status.py | 55 +++ cli/canyonos/stop.py | 10 +- cli/canyonos/sync.py | 19 +- cli/canyonos/test.py | 365 ++++++++++++--- cli/canyonos/ui.py | 67 +++ cli/canyonos/verify.py | 291 ++++++++++++ cli/cli.py | 100 ++-- cli/pyproject.toml | 2 +- cli/utils/help_screen.py | 60 ++- cli/utils/tui.py | 8 +- examples/finance/agents/finance_agent.py | 9 +- examples/finance/workflow/example_workflow.py | 8 +- examples/helloworld/README.md | 4 +- .../helloworld/workflow/example_workflow.py | 6 +- examples/portfolio/agents/metrics_agent.py | 14 +- .../text2sql/agents/sql_generator_agent.py | 9 +- .../text2sql/workflow/text2sql_workflow.py | 12 +- pyproject.toml | 5 + tests/test_canyonos_test.py | 426 ++++++++++++++++++ tests/test_dashboard_stack.py | 146 +----- tests/test_deploy_progress.py | 235 ++++++++++ tests/test_integration.py | 2 +- uv.lock | 77 +++- .../cloud_provider_logic/EC2/_runtime.py | 11 +- .../cloud_provider_logic/Local/_runtime.py | 2 + ventis/controller/instance_manager.py | 9 +- ventis/server.py | 80 +++- 40 files changed, 2213 insertions(+), 570 deletions(-) create mode 100644 cli/canyonos/status.py create mode 100644 cli/canyonos/ui.py create mode 100644 cli/canyonos/verify.py create mode 100644 tests/test_canyonos_test.py create mode 100644 tests/test_deploy_progress.py diff --git a/README.md b/README.md index 1d61db8..0f1ea27 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ The Readme in the newly created project directory provides a quick overview of t #### Step 2: Define Your Agents Agent declarations live under `.car/config/`. The source used for builds is -copied to `.car/app/` by `canyonos integrate`. +copied to `.car/app/` by `canyonos build`. - **`.car/config/my_agent.yaml`**: Defines methods and schemas. - **`.car/app/path/to/my_agent.py`**: Contains the agent implementation. @@ -102,7 +102,7 @@ Users can send requests to this endpoint to trigger the workflow. For this examp curl -X POST http://localhost:8080/main \ -H "Content-Type: application/json" \ -d '{ - "ticker": "AAPL" + "query": "AAPL" }' ``` The request is asynchronous. To get the result, you use the following URL- diff --git a/cli/canyonos/build.py b/cli/canyonos/build.py index f5292aa..c9b56b4 100644 --- a/cli/canyonos/build.py +++ b/cli/canyonos/build.py @@ -10,8 +10,7 @@ import tempfile import urllib.request -from rich.console import Console - +from canyonos import ui from utils.tui import select_menu SKILL_OWNER = "CanyonCodeCoreAI" @@ -43,19 +42,24 @@ " the endpoint must keep the /v1/traces path:\n\n" + OTEL_BLOCK ) +# The leaf name of every install path must match the skill's own `name:` +# frontmatter or the agent won't resolve it. AGENTS = { "claude": { "label": "Claude Code", "cli": "claude", - # Claude Code auto-loads project-local skills from here. The leaf name - # must match the skill's own `name:` frontmatter or it won't resolve. - "skill_dir": SKILL_PATH, + "skill_dirs": { + "local": SKILL_PATH, + "global": os.path.expanduser(f"~/.claude/skills/{SKILL_NAME}"), + }, }, "codex": { "label": "Codex", "cli": "codex", - # Codex only auto-loads skills from the user's home directory, not per-project. - "skill_dir": os.path.expanduser(f"~/.codex/skills/{SKILL_NAME}"), + "skill_dirs": { + "local": f".codex/skills/{SKILL_NAME}", + "global": os.path.expanduser(f"~/.codex/skills/{SKILL_NAME}"), + }, }, } @@ -65,6 +69,15 @@ def prompt_agent(): return select_menu(options, title="Which coding agent do you want to build on?") +def prompt_scope(agent): + dirs = AGENTS[agent]["skill_dirs"] + options = [ + ("local", f"This project only ({dirs['local']})"), + ("global", f"Globally ({dirs['global']})"), + ] + return select_menu(options, title="Where should the CanyonOS skill be installed?") + + def _replace_dir(source, dest): """Move `source` onto `dest`, replacing whatever was there.""" os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) @@ -143,47 +156,32 @@ def _fetch_with_tarball(dest): return True -def _fetch_with_npx(dest): - """Last resort, and the only strategy that needs Node.""" - if not shutil.which("npx"): - return False - # -f overwrites an existing skill dir; without it gitpick exits 1 when the - # target already exists and is non-empty (e.g. re-running `build`). - return subprocess.run( - ["npx", "-y", "gitpick", "-f", TREE_URL, dest], capture_output=True - ).returncode == 0 - - FETCH_STRATEGIES = ( ("git", _fetch_with_git), ("tarball", _fetch_with_tarball), - ("npx", _fetch_with_npx), ) -def install_skill(agent, console): - """Fetch the skill into the agent's skill dir. Returns True on success.""" - dest = AGENTS[agent]["skill_dir"] +def install_skill(dest): + """Fetch the skill into `dest`. Returns True on success.""" for name, fetch in FETCH_STRATEGIES: try: if fetch(dest): - console.print(f"Fetched the CanyonOS skill via {name}.") + ui.ok(f"Fetched the CanyonOS skill via {name}.") return True except OSError: pass - console.print(f"[dim]{name} fetch unavailable, trying the next option...[/dim]") + ui.hint(f"{name} fetch unavailable, trying the next option...") - console.print( - f"Could not fetch the CanyonOS skill from {TREE_URL}.\n" - "Install git or Node, or check network access, then run `canyonos doctor`." - ) + ui.fail(f"Could not fetch the CanyonOS skill from {TREE_URL}.") + ui.hint("Install git, or check network access, then run `canyonos doctor`.") return False def launch_agent(agent, prompt): spec = AGENTS[agent] if not shutil.which(spec["cli"]): - print(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") + ui.fail(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") return # No check=True: the agent exiting non-zero (including the user quitting it) # is an ordinary outcome, not something to raise a traceback over. @@ -191,15 +189,20 @@ def launch_agent(agent, prompt): def run_build(): - console = Console() agent = prompt_agent() if agent is None: - console.print("Cancelled.") + ui.say("Cancelled.") + return + + scope = prompt_scope(agent) + if scope is None: + ui.say("Cancelled.") return - console.print(f"Installing CanyonOS skill for {AGENTS[agent]['label']}...") - if not install_skill(agent, console): + dest = AGENTS[agent]["skill_dirs"][scope] + ui.say(f"Installing CanyonOS skill for {AGENTS[agent]['label']} into {dest}...") + if not install_skill(dest): return - console.print(f"Launching {AGENTS[agent]['label']}...") + ui.say(f"Launching {AGENTS[agent]['label']}...") launch_agent(agent, BUILD_PROMPT) diff --git a/cli/canyonos/clean.py b/cli/canyonos/clean.py index d7cc6f5..9c974b4 100644 --- a/cli/canyonos/clean.py +++ b/cli/canyonos/clean.py @@ -5,14 +5,16 @@ import os import shutil +from canyonos import ui + def run_clean(): car_dir = os.path.join(os.getcwd(), ".car") if not os.path.isdir(car_dir): - print("Nothing to clean, no .car folder in root") + ui.warn("Nothing to clean, no .car folder in root") return - print(f"Cleaning {car_dir}...") - shutil.rmtree(car_dir) - print("Clean complete.") + with ui.status(f"Cleaning {car_dir}..."): + shutil.rmtree(car_dir) + ui.ok("Clean complete.") diff --git a/cli/canyonos/config.py b/cli/canyonos/config.py index 25d2d3d..aec5b12 100644 --- a/cli/canyonos/config.py +++ b/cli/canyonos/config.py @@ -5,11 +5,11 @@ import os import yaml -from rich.console import Console from rich.table import Table from canyonos.constants import default_config_path, round_trip_yaml from canyonos.theme import GREEN, WHITE +from canyonos import ui from utils.tui import DELETE_ACTION, QUIT_ACTION, select_menu BACK = "__back__" @@ -98,30 +98,29 @@ def _kv_table(title, data): return table -def _require_config(config_path, console): +def _require_config(config_path): """Resolved config path, or None after reporting that it's missing.""" config_path = config_path or default_config_path() if not os.path.isfile(config_path): - console.print(f"[red]Config file not found: {config_path}[/red]") + ui.fail(f"Config file not found: {config_path}") return None return config_path def run_view_config(config_path=None): - console = Console() - config_path = _require_config(config_path, console) + config_path = _require_config(config_path) if config_path is None: return with open(config_path) as f: config = yaml.safe_load(f) or {} - console.print(_agents_table(config.get("agents") or [])) - console.print() + ui.console.print(_agents_table(config.get("agents") or [])) + ui.blank() if config.get("otel"): - console.print(_otel_table(config["otel"])) - console.print() + ui.console.print(_otel_table(config["otel"])) + ui.blank() # Every other top-level key: dicts get their own table, bare scalars are # gathered into a single "General" table. @@ -130,13 +129,13 @@ def run_view_config(config_path=None): if key in STRUCTURED_KEYS: continue if isinstance(value, dict): - console.print(_kv_table(key, value)) - console.print() + ui.console.print(_kv_table(key, value)) + ui.blank() else: general[key] = value if general: - console.print(_kv_table("General", general)) + ui.console.print(_kv_table("General", general)) def _is_leaf(value): @@ -294,8 +293,7 @@ def _navigate(screen, node, breadcrumb): def run_change_config(config_path=None): - console = Console() - config_path = _require_config(config_path, console) + config_path = _require_config(config_path) if config_path is None: return @@ -304,14 +302,14 @@ def run_change_config(config_path=None): data = yaml_rt.load(f) if not data: - console.print("[yellow]Config is empty; nothing to change.[/yellow]") + ui.warn("Config is empty; nothing to change.") return - screen = _Screen(console) + screen = _Screen(ui.console) saves = 0 # Alternate screen: the whole session replaces the view, and the terminal # scrollback is restored untouched on exit. - console.set_alt_screen(True) + ui.console.set_alt_screen(True) try: while True: changed = _navigate(screen, data, ["config"]) @@ -323,19 +321,18 @@ def run_change_config(config_path=None): saves += 1 screen.status = f"Saved to {config_path}" finally: - console.set_alt_screen(False) + ui.console.set_alt_screen(False) if saves: - console.print(f"[{GREEN}]Saved {saves} change(s) to {config_path}[/]") + ui.ok(f"Saved {saves} change(s) to {config_path}") else: - console.print("No changes made.") + ui.say("No changes made.") def run_config(): - console = Console() choice = select_menu(OPTIONS, title="What do you want to do?") if choice is None: - console.print("Cancelled.") + ui.say("Cancelled.") return if choice == "view": diff --git a/cli/canyonos/dashboard_stack.py b/cli/canyonos/dashboard_stack.py index c606357..e26fbcf 100644 --- a/cli/canyonos/dashboard_stack.py +++ b/cli/canyonos/dashboard_stack.py @@ -10,7 +10,6 @@ import shutil import socket import subprocess -import tempfile import time import urllib.error import urllib.request @@ -39,11 +38,10 @@ class ServeResult: class PhaseFailure(Exception): - def __init__(self, phase: str, message: str, *, had_containers: bool | None = None): + def __init__(self, phase: str, message: str): super().__init__(message) self.phase = phase self.message = message - self.had_containers = had_containers @dataclass(frozen=True) @@ -179,8 +177,14 @@ def _read_existing_secret(env_path: Path) -> str | None: def _write_private_file(path: Path, contents: str) -> None: + """Write 0600 from the start, so the contents are never briefly world-readable. + + The open mode only applies when creating, so an already-loose file (a `.env` + the user wrote by hand) is tightened explicitly rather than left as it was. + """ descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(descriptor, "w", encoding="utf-8") as output: + os.fchmod(output.fileno(), 0o600) output.write(contents) @@ -191,40 +195,27 @@ def _env_line(key: str, value: str) -> str: def _write_project_env(env_path: Path, managed_env: dict[str, str]) -> None: + """Rewrite only the CANYONOS_* keys, leaving every other line of the user's .env alone.""" try: lines = env_path.read_text(encoding="utf-8").splitlines(keepends=True) except FileNotFoundError: lines = [] - managed_keys = set(managed_env) replaced: set[str] = set() updated_lines: list[str] = [] for line in lines: key, separator, _ = line.partition("=") - if separator and key in managed_keys: + if separator and key in managed_env: if key not in replaced: updated_lines.append(_env_line(key, managed_env[key])) replaced.add(key) continue updated_lines.append(line) - for key, value in managed_env.items(): - if key not in replaced: - updated_lines.append(_env_line(key, value)) - - descriptor, temporary_name = tempfile.mkstemp(prefix=".env.", dir=env_path.parent) - temporary_path = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as output: - os.fchmod(output.fileno(), 0o600) - output.writelines(updated_lines) - os.replace(temporary_path, env_path) - except Exception: - try: - temporary_path.unlink() - except FileNotFoundError: - pass - raise + updated_lines.extend( + _env_line(key, value) for key, value in managed_env.items() if key not in replaced + ) + _write_private_file(env_path, "".join(updated_lines)) def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: @@ -257,36 +248,27 @@ def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: return managed_env, "dashboard state prepared" -def _last_stderr_line(result: subprocess.CompletedProcess[str]) -> str | None: - return next((line.strip() for line in reversed(result.stderr.splitlines()) if line.strip()), None) - - def _command_failure_message( message: str, result: subprocess.CompletedProcess[str], managed_env: dict[str, str], ) -> str: - detail = _last_stderr_line(result) + detail = next( + (line.strip() for line in reversed(result.stderr.splitlines()) if line.strip()), None + ) if detail is None: return message return f"{message}: {redact_logs(detail, managed_env['CANYONOS_JWT_SECRET'])}" -def pull( - stack: DashboardStack, - manifest: Path, - managed_env: dict[str, str], - had_containers: bool, -) -> str: +def pull(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> str: try: result = _run([*_compose_argv(stack, manifest), "pull"]) except OSError: - raise PhaseFailure("pull", "could not run docker compose pull", had_containers=had_containers) + raise PhaseFailure("pull", "could not run docker compose pull") if result.returncode != 0: raise PhaseFailure( - "pull", - _command_failure_message("docker compose pull failed", result, managed_env), - had_containers=had_containers, + "pull", _command_failure_message("docker compose pull failed", result, managed_env) ) return "dashboard images pulled" @@ -299,8 +281,7 @@ def _project_has_running_containers(stack: DashboardStack, manifest: Path) -> bo return result.returncode == 0 and bool(result.stdout.strip()) -def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> bool: - had_containers = _project_has_running_containers(stack, manifest) +def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> None: # The api reads the controller's Redis identity once at startup to create # its project row, so a surviving container keeps serving whichever project # was deployed before it. Replace it every serve rather than reuse it. @@ -310,14 +291,11 @@ def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> [*_compose_argv(stack, manifest), "up", "-d", "--wait", "--wait-timeout", "180"] ) except OSError: - raise PhaseFailure("start", "could not run docker compose up", had_containers=had_containers) + raise PhaseFailure("start", "could not run docker compose up") if result.returncode != 0: raise PhaseFailure( - "start", - _command_failure_message("docker compose up failed", result, managed_env), - had_containers=had_containers, + "start", _command_failure_message("docker compose up failed", result, managed_env) ) - return had_containers def verify(port: int) -> str: @@ -386,6 +364,8 @@ def report(result: ServeResult) -> None: stack: DashboardStack | None = None managed_env: dict[str, str] | None = None manifest: Path | None = None + # Whether the stack predates this serve, so a failure only tears down what + # this run brought up. Read once, before anything here can change it. had_containers = False with ExitStack() as resources: try: @@ -397,11 +377,11 @@ def report(result: ServeResult) -> None: manifest_resource = importlib.resources.files("canyonos").joinpath("dashboard.compose.yml") manifest = resources.enter_context(importlib.resources.as_file(manifest_resource)) - had_containers_before_pull = _project_has_running_containers(stack, manifest) - pull_message = pull(stack, manifest, managed_env, had_containers_before_pull) - report(ServeResult(True, "pull", pull_message)) + had_containers = _project_has_running_containers(stack, manifest) + + report(ServeResult(True, "pull", pull(stack, manifest, managed_env))) - had_containers = start(stack, manifest, managed_env) + start(stack, manifest, managed_env) report(ServeResult(True, "start", "dashboard stack started")) url = verify(stack.web_port) @@ -411,7 +391,7 @@ def report(result: ServeResult) -> None: log_path = None if failure.phase in {"pull", "start", "verify"} and stack and managed_env and manifest: log_path = _capture_failure_logs(stack, manifest, managed_env) - if not (failure.had_containers if failure.had_containers is not None else had_containers): + if not had_containers: _cleanup(stack, manifest) return ServeResult( False, failure.phase, failure.message, None, str(log_path) if log_path else None diff --git a/cli/canyonos/deploy.py b/cli/canyonos/deploy.py index 57c32ad..048cd3d 100644 --- a/cli/canyonos/deploy.py +++ b/cli/canyonos/deploy.py @@ -3,43 +3,162 @@ volume (via `canyonos sync`), then tell the Global Controller container to build and deploy it. The container's `ventis deploy` handles both the build (stubs, protos, Docker images) and the launch -- the CLI just ships files, -triggers it, and streams the logs. +triggers it, and watches the logs. + +That log stream is mostly noise the user didn't ask for (a whole `docker buildx +bake` transcript, among other things), so by default only the phase transitions +worth seeing are rendered and everything else is dropped. `-v` streams it all, +and a failure reveals the output it had been hiding. Once the deploy's logs report the workflow is actually up, `canyonos serve` is kicked off automatically so the local dashboard is ready without an extra manual step. """ +import queue +import re import subprocess +import threading +import time +from collections import deque -from rich.console import Console from rich.panel import Panel from rich.text import Text +from canyonos import ui from canyonos.constants import ( WORKFLOW_ROUTE, default_config_path, workflow_api_port, workspace_relative, ) -from canyonos.gc import GCError, post_deploy +from canyonos.gc import GCError, deploy_status, post_deploy, workflow_endpoints from canyonos.theme import GREEN, WHITE from canyonos.init import load_state, run_init -from canyonos.serve import run_serve +from canyonos.serve import serve_dashboard from canyonos.sync import run_sync +LOCAL_HOSTS = ("127.0.0.1", "localhost") + # Logged exactly once by GlobalController.run(), right after `_wait_for_healthy()` # returns -- the signal that the workflow finished coming up and entered its # steady-state polling loop. _WORKFLOW_UP_MARKER = "Global controller started, polling every" +# Substrings that mean the in-container deploy hit something fatal. `WARNING:` is +# deliberately absent: the OTel-not-configured notice and stub_generator's +# "Warning:" lines are benign and fire on nearly every run. +_ERROR_MARKERS = ( + "ERROR:", + "Traceback (most recent call last):", + "ERROR: failed to solve", + "process did not complete successfully", +) + +# (substring, spinner message, completed message). A None spinner message keeps +# whatever the spinner already shows; a None completed message prints nothing. +# Matched by substring against the raw line, so a phase that never runs is simply +# never matched -- nothing here assumes a phase happens, or happens in order. +_PHASES = ( + ("Generating stub:", "Generating stubs and Docker contexts...", None), + ("Compiling gRPC proto:", "Generating stubs and Docker contexts...", None), + ("Generating Docker context", "Generating stubs and Docker contexts...", None), + ("Building Docker image:", "Building images...", None), + ("No Docker images to build.", None, "No images to build"), + ("Build complete.", None, "Build complete"), + ("Deploying from config:", "Starting deploy...", None), + ("Checking for stale containers", "Cleaning up stale containers...", None), + ("Redis launched on", None, "Redis ready"), + ("Docker container(s) across", "Starting agents...", None), +) + +_IMAGE_COUNT = re.compile(r"Building (\d+) Docker image\(s\) via") +_REPLICA_COUNT = re.compile(r"Waiting for (\d+) replica\(s\) to become healthy") +# The name repeats across replicas of one agent, so the endpoint is what makes a +# ready line unique. +_READY = re.compile(r"Controller (\S+ \([^)]+\)) is ready\.") + +# Enough to hold a buildx failure block plus a Python traceback; 40 (what +# `canyonos test` tails) truncates both. +_RECENT_LINES = 200 + +# The container logs every request the CLI makes to it, so its own polling shows +# up in the stream it is reading. +_OWN_REQUEST_MARKER = "GET /status HTTP/1.1" + +_STATUS_POLL_SECONDS = 2.0 + +# Upper bound on how long to keep collecting output after a failure is spotted. +_REVEAL_GRACE_SECONDS = 30.0 + + +class PhaseTracker: + """Turns the container's log lines into the handful of events worth showing. + + `feed()` returns (spinner_message, completed_message, is_error) -- any of + which may be None -- so the caller owns all printing. + """ + + def __init__(self): + self.spinner = None + self.replicas_total = 0 + self.replicas_ready = set() + + def _agent_progress(self): + if self.replicas_total: + return f"Starting agents ({len(self.replicas_ready)}/{self.replicas_total} ready)..." + return "Starting agents..." + + def feed(self, line): + if any(marker in line for marker in _ERROR_MARKERS): + return None, None, True + + count = _IMAGE_COUNT.search(line) + if count: + self.spinner = f"Building {count.group(1)} images..." + return self.spinner, None, False + + replicas = _REPLICA_COUNT.search(line) + if replicas: + self.replicas_total = int(replicas.group(1)) + self.spinner = self._agent_progress() + return self.spinner, None, False + + ready = _READY.search(line) + if ready: + self.replicas_ready.add(ready.group(1)) + self.spinner = self._agent_progress() + return self.spinner, None, False -def run_deploy(config_path=None, serve=True): + for marker, spinner, done in _PHASES: + if marker in line: + # Repeats (one `Generating stub:` per agent) collapse: the + # spinner is only re-emitted when the message actually changes. + if spinner and spinner != self.spinner: + self.spinner = spinner + return spinner, done, False + return None, done, False + + return None, None, False + + def agents_ready_message(self): + """(message, all_ready). `_wait_for_healthy` gives up after its timeout and + lets the controller start anyway, so the workflow can come up short. + """ + ready = len(self.replicas_ready) + if not self.replicas_total: + return "Workflow ready", True + if ready < self.replicas_total: + return f"Workflow up, but only {ready}/{self.replicas_total} agents reported healthy", False + return f"{ready} agent(s) ready", True + + +def run_deploy(config_path=None, serve=True, verbose=False): # Left as None when unset: ventis resolves the artifact layout itself. if config_path is not None: config_path = workspace_relative(config_path) if config_path is None: - print("Config must be inside the project directory being synced.") + ui.fail("Config must be inside the project directory being synced.") return run_init() @@ -55,76 +174,254 @@ def run_deploy(config_path=None, serve=True): try: post_deploy(state["port"], config_path) - _stream_logs_and_autoserve(state["container_id"], api_port, serve=serve) + _stream_logs_and_autoserve(state, api_port, serve=serve, verbose=verbose) except GCError as e: - print(e) + ui.fail(e) -def print_workflow_endpoint(console, api_port): - """The one thing you need after a deploy: where to send requests. +def workflow_targets(gc_port, api_port): + """(name, host, port) for each deployed workflow. - Printed at the workflow-up marker and again on exit, because `deploy` keeps - tailing logs afterwards and would otherwise scroll it out of sight. + The container reports the address it actually placed each workflow at, so a + workflow running on another machine shows that machine's public IP. The + local port mapping is the fallback when it reports nothing. """ - if api_port is None: - return + targets = [ + ( + endpoint.get("name"), + "127.0.0.1" if endpoint["host"] in LOCAL_HOSTS else endpoint["host"], + endpoint["port"], + ) + for endpoint in workflow_endpoints(gc_port) + if endpoint.get("host") and endpoint.get("port") + ] + if targets: + return targets + return [(None, "127.0.0.1", api_port)] if api_port else [] - url = f"http://127.0.0.1:{api_port}/{WORKFLOW_ROUTE}" - body = Text.assemble( - ("POST ", "dim"), - (url, f"bold {GREEN}"), - ("\nbody ", "dim"), - ('{"query": "your question here"}', WHITE), - ("\npoll ", "dim"), - (f"http://127.0.0.1:{api_port}/status/", WHITE), - ) - console.print() - console.print( + +def _summary_body(dashboard_url, targets): + body = Text() + body.append("Dashboard ", "dim") + if dashboard_url: + body.append(dashboard_url, f"bold {GREEN}") + else: + body.append("not running -- start it with `canyonos serve`", WHITE) + + for name, host, port in targets: + base = f"http://{host}:{port}" + body.append("\n") + if name: + body.append(f"\n{name}", f"bold {WHITE}") + body.append("\nPOST ", "dim") + body.append(f"{base}/{WORKFLOW_ROUTE}", f"bold {GREEN}") + body.append("\nbody ", "dim") + body.append('{"query": "your question here"}', WHITE) + body.append("\npoll ", "dim") + body.append(f"{base}/status/", WHITE) + if host not in LOCAL_HOSTS: + body.append(f"\n needs inbound TCP {port} open on {host}", "dim") + return body + + +def print_deploy_summary(dashboard_url, targets): + """The one screen printed once everything is up: dashboard and workflow endpoints. + + Under `-v` it is printed again on exit, because the log tail continues + afterwards and would otherwise scroll it out of sight. Quiet mode prints + nothing after it, so once is enough. + """ + ui.blank() + ui.panel( Panel( - body, - title=f"[bold {GREEN}]Workflow is live[/]", + _summary_body(dashboard_url, targets), + title=f"[bold {GREEN}]Deploy is live[/]", title_align="left", border_style=GREEN, padding=(1, 4), ) ) - console.print() + ui.blank() + + +def _start_dashboard(): + """The dashboard's URL, or None -- a dashboard that won't start doesn't fail the deploy.""" + try: + return serve_dashboard().url + except Exception as e: + ui.fail(f"Could not start the dashboard automatically: {e}") + ui.hint("Run `canyonos serve` manually to view it.") + return None + + +def _deploy_summary(state, api_port, serve): + summary = ( + _start_dashboard() if serve else None, + workflow_targets(state["port"], api_port), + ) + print_deploy_summary(*summary) + return summary -def _stream_logs_and_autoserve(container_id, api_port, serve=True): +def _interrupted(summary=None): + ui.blank() + ui.say("Stopped monitoring log stream. Run `canyonos stop` to stop the deploy.") + ui.hint("To resubscribe to log stream run `canyonos logs`.") + if summary is not None: + print_deploy_summary(*summary) + + +def _tail_verbose(stream, state, api_port, serve): + """Every log line, verbatim -- what `-v` restores. + + Ctrl+C reprints the summary here but not in quiet mode: only this tail keeps + printing past it, so only here has it scrolled out of sight. + """ + summary = None + try: + for line in stream: + print(line, end="") + if summary is None and _WORKFLOW_UP_MARKER in line: + summary = _deploy_summary(state, api_port, serve) + except KeyboardInterrupt: + _interrupted(summary) + + +def _tail_quiet(lines, state, api_port, serve): + """Only the phase transitions, until the workflow is up or something fails. + + Nothing is echoed raw: the buildx transcript, ventis' bare prints and grpc's + stderr have no common prefix to filter on, so anything unrecognized is + dropped rather than allow-listed. `-v` and `canyonos logs` still have it all. + """ + tracker = PhaseTracker() + recent = deque(maxlen=_RECENT_LINES) + reached_up_marker = False + + # The spinner is exited before the summary panel or the dashboard's own + # spinner is drawn, and on the way out of a Ctrl+C, so the cursor is restored. + # A nested spinner wouldn't raise, it would silently render nothing. + with ui.status("Starting build...") as spinner: + for line in _drain(lines, state): + recent.append(line) + message, done, is_error = tracker.feed(line) + if is_error: + break + if done: + ui.ok(done) + if message: + spinner.update(message) + if _WORKFLOW_UP_MARKER in line: + summary_line, all_ready = tracker.agents_ready_message() + (ui.ok if all_ready else ui.warn)(summary_line) + reached_up_marker = True + break + + if reached_up_marker: + return _deploy_summary(state, api_port, serve) + + _reveal_failure(lines, recent, state) + return None + + +def _queued_lines(stream): + """Feed `stream` into a queue, terminated by None, so reads can time out. + + A failed build leaves the log stream open and silent -- the deploy is only a + subprocess of the container being tailed -- so blocking on the next line + would wait forever with nothing left to report. + """ + lines = queue.Queue() + + def read(): + for line in stream: + lines.put(line) + lines.put(None) + + threading.Thread(target=read, daemon=True).start() + return lines + + +def _drain(lines, state, deadline=None): + """Yield log lines until the stream ends, the deploy dies, or `deadline` passes. + + The container's /status is polled on the read timeout rather than per line, + because the container logs each of those requests into the very stream being + read -- which would otherwise feed itself. + """ + misses = 0 + while deadline is None or time.monotonic() < deadline: + try: + line = lines.get(timeout=_STATUS_POLL_SECONDS) + except queue.Empty: + # Nothing for a while: check the deploy is still alive, since a + # build that died takes the output with it but not the stream. + dead, misses = _deploy_is_dead(state, misses) + if dead: + return + continue + if line is None: + return + misses = 0 + if _OWN_REQUEST_MARKER not in line: + yield line + + +def _deploy_is_dead(state, misses): + """Whether the in-container deploy has stopped, over two consecutive checks. + + An unreachable container counts as a miss rather than a verdict, so one + dropped request doesn't end a deploy that is merely busy. + """ + status = deploy_status(state["port"]) + if status is not None and status.get("running"): + return False, 0 + misses += 1 + return misses >= 2, misses + + +def _reveal_failure(lines, recent, state): + """Stop hiding: replay what was suppressed, then keep echoing. + + The cause is usually still in flight when the verdict lands, so this keeps + draining until the container confirms the deploy is gone. + """ + ui.fail("Deploy failed.") + ui.blank() + for buffered in recent: + print(buffered, end="") + + for line in _drain(lines, state, deadline=time.monotonic() + _REVEAL_GRACE_SECONDS): + print(line, end="") + + ui.blank() + ui.hint("Run `canyonos deploy -v` or `canyonos logs` for the full container log.") + + +def _stream_logs_and_autoserve(state, api_port, serve=True, verbose=False): """Tail the GC container's logs, and once they show the workflow is up, - print where to reach it -- plus, unless disabled via `serve=False`, launch - `canyonos serve`. Log tailing continues afterwards. + start the dashboard (unless disabled via `serve=False`) and print where + everything lives. Log tailing continues afterwards. """ - console = Console() process = subprocess.Popen( - ["docker", "logs", "-f", container_id], + ["docker", "logs", "-f", state["container_id"]], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, ) - served = not serve - workflow_up = False try: - for line in process.stdout: - print(line, end="") - if not workflow_up and _WORKFLOW_UP_MARKER in line: - workflow_up = True - print_workflow_endpoint(console, api_port) - if not served: - served = True - print("Starting the local dashboard (canyonos serve)...") - try: - run_serve() - except Exception as e: - print(f"Could not start the dashboard automatically: {e}") - print("Run `canyonos serve` manually to view it.") + if verbose: + _tail_verbose(process.stdout, state, api_port, serve) + return + lines = _queued_lines(process.stdout) + if _tail_quiet(lines, state, api_port, serve) is not None: + # Quiet mode stays attached after the summary so Ctrl+C means the + # same thing in both modes -- it just swallows what arrives. + while lines.get() is not None: + pass except KeyboardInterrupt: - print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") - print("To resubscribe to log stream run `canyonos logs`.") - if workflow_up: - print_workflow_endpoint(console, api_port) + _interrupted() finally: if process.poll() is None: process.terminate() diff --git a/cli/canyonos/doctor.py b/cli/canyonos/doctor.py index 7dd9883..15c36af 100644 --- a/cli/canyonos/doctor.py +++ b/cli/canyonos/doctor.py @@ -7,17 +7,9 @@ import shutil import subprocess +from canyonos import ui from canyonos.build import AGENTS -from canyonos.init import docker_start_command - - -def _docker_installed(): - return shutil.which("docker") is not None - - -def _docker_daemon_running(): - result = subprocess.run(["docker", "info"], capture_output=True) - return result.returncode == 0 +from canyonos.init import docker_running, docker_start_command def _compose_available(): @@ -25,10 +17,6 @@ def _compose_available(): return result.returncode == 0 -def _git_available(): - return shutil.which("git") is not None - - def _docker_daemon_fix(): """Names the command for the active docker context, since `canyonos deploy` would run exactly that itself.""" @@ -38,23 +26,16 @@ def _docker_daemon_fix(): return "start your Docker runtime (on Linux: `sudo systemctl start docker`)" -def _coding_agent_available(): - return any(shutil.which(spec["cli"]) for spec in AGENTS.values()) - - def _checks(): - """Built fresh on each call (not a module-level constant) so tests can - patch the individual `_check_*` functions by name and have it take effect. - """ return [ ( "Docker installed", - _docker_installed, + lambda: shutil.which("docker") is not None, "install Docker: https://docs.docker.com/get-docker/", ), ( "Docker daemon running", - _docker_daemon_running, + docker_running, _docker_daemon_fix(), ), ( @@ -64,13 +45,13 @@ def _checks(): ), ( "git available", - _git_available, + lambda: shutil.which("git") is not None, "install git (`canyonos build` fetches the porting skill with it; " "without git it falls back to a full-repo tarball download)", ), ( "Coding agent available", - _coding_agent_available, + lambda: any(shutil.which(spec["cli"]) for spec in AGENTS.values()), "install one of " + " or ".join(spec["label"] for spec in AGENTS.values()) + " (`canyonos build` runs the port through it)", @@ -88,9 +69,11 @@ def run_doctor(): passed = False fix = f"{fix} (error: {e})" - print(f"{'✓' if passed else '✗'} {label}") - if not passed: - print(f" -> {fix}") + if passed: + ui.ok(label) + else: + ui.fail(label) + ui.hint(f" -> {fix}") all_ok = False return all_ok diff --git a/cli/canyonos/gc.py b/cli/canyonos/gc.py index b5af7eb..a8778b3 100644 --- a/cli/canyonos/gc.py +++ b/cli/canyonos/gc.py @@ -7,6 +7,7 @@ import urllib.error import urllib.request +from canyonos import ui from canyonos.init import load_state _DEPLOY_CONFLICT = "Run `canyonos stop` to stop the running deploy first." @@ -46,7 +47,7 @@ def require_state(): try: return load_state() except FileNotFoundError: - print("No Global Controller container is running. Run `canyonos deploy` first.") + ui.warn("No Global Controller container is running. Run `canyonos deploy` first.") return None @@ -73,6 +74,19 @@ def post_clean(port): return _request(f"http://127.0.0.1:{port}/clean", "Stop", method="POST") +def workflow_endpoints(port): + """Where the deployed workflows answer, per the container's own instance + records -- for a workflow placed on another machine that is its public IP, + not this host. Empty when the container can't say (an older image has no + /endpoints route), which leaves the caller on its local-port fallback. + """ + try: + data = _request(f"http://127.0.0.1:{port}/endpoints", "Endpoints") + except GCError: + return [] + return data.get("workflows") or [] + + def deploy_status(port): """Parsed /status payload, or None if the container is unreachable.""" url = f"http://127.0.0.1:{port}/status" diff --git a/cli/canyonos/init.py b/cli/canyonos/init.py index 69e8fbb..6b07f84 100644 --- a/cli/canyonos/init.py +++ b/cli/canyonos/init.py @@ -16,9 +16,8 @@ # Formatting from pyfiglet import figlet_format -from rich.console import Console -from canyonos.theme import GRADIENT +from canyonos import ui @@ -76,7 +75,7 @@ def docker_start_command(): return None -def ensure_docker_running(console, timeout=DOCKER_START_TIMEOUT): +def ensure_docker_running(timeout=DOCKER_START_TIMEOUT): if docker_running(): return @@ -89,14 +88,14 @@ def ensure_docker_running(console, timeout=DOCKER_START_TIMEOUT): "docker context. Start it (on Linux: `sudo systemctl start docker`) and re-run." ) - console.print(f"Docker isn't running -- starting it with `{' '.join(command)}`...") + ui.say(f"Docker isn't running -- starting it with `{' '.join(command)}`...") subprocess.run(command, capture_output=True) deadline = time.time() + timeout - with console.status("Waiting for the Docker daemon..."): + with ui.status("Waiting for the Docker daemon..."): while time.time() < deadline: if docker_running(): - console.print("Docker is running.") + ui.ok("Docker is running.") return time.sleep(1) @@ -200,20 +199,17 @@ def quit_existing(): run_quit() -def run_init(): - console = Console() - banner = figlet_format("CANYON OS", font="ansi_shadow", width=200) - - for line, color in zip(banner.splitlines(), GRADIENT): - console.print(line, style=color) +def run_init(banner=True): + if banner: + ui.gradient(figlet_format("CANYON OS", font="ansi_shadow", width=200)) # Before quit_existing(), which shells out to docker itself. - ensure_docker_running(console) + ensure_docker_running() quit_existing() - with console.status("Pulling Global Controller image..."): + with ui.status("Pulling Global Controller image..."): pull_image() - with console.status("Starting Global Controller container..."): + with ui.status("Starting Global Controller container..."): container_id, port = run_container() save_state(container_id, port) - print(f"Global Controller running in container {container_id[:12]} on port {port}") + ui.ok(f"Global Controller running in container {container_id[:12]} on port {port}") diff --git a/cli/canyonos/logs.py b/cli/canyonos/logs.py index 9839134..3969af6 100644 --- a/cli/canyonos/logs.py +++ b/cli/canyonos/logs.py @@ -4,6 +4,7 @@ import subprocess +from canyonos import ui from canyonos.gc import deploy_status, require_state @@ -14,14 +15,15 @@ def run_logs(): status = deploy_status(state["port"]) if status is None: - print("Could not reach Global Controller container.") + ui.fail("Could not reach Global Controller container.") return if not status.get("running"): - print("No deploy running, run `canyonos deploy` to deploy project.") + ui.warn("No deploy running, run `canyonos deploy` to deploy project.") return try: subprocess.run(["docker", "logs", "-f", state["container_id"]]) except KeyboardInterrupt: - print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") + ui.blank() + ui.say("Stopped monitoring log stream. Run `canyonos stop` to stop the deploy.") diff --git a/cli/canyonos/new_app.py b/cli/canyonos/new_app.py index 30e93b0..31aeb44 100644 --- a/cli/canyonos/new_app.py +++ b/cli/canyonos/new_app.py @@ -5,10 +5,12 @@ import os +from canyonos import ui + def run_new_app(): if os.listdir("."): - print("Directory is not empty. Run `canyonos new-app` in an empty directory.") + ui.fail("Directory is not empty. Run `canyonos new-app` in an empty directory.") return for folder in ("agents", "config", "workflow"): @@ -18,4 +20,4 @@ def run_new_app(): for filename in ("global_controller.yaml", "policy.yaml"): open(os.path.join("config", filename), "w").close() - print("Created new CanyonOS project.") + ui.ok("Created new CanyonOS project.") diff --git a/cli/canyonos/quit.py b/cli/canyonos/quit.py index bb2e11d..9af5dcc 100644 --- a/cli/canyonos/quit.py +++ b/cli/canyonos/quit.py @@ -8,8 +8,7 @@ import os import subprocess -from rich.console import Console - +from canyonos import ui from canyonos.gc import GCError, post_clean, require_state from canyonos.init import GC_WORKSPACE_VOLUME, STATE_PATH @@ -27,8 +26,7 @@ def run_quit(): return container_id = state["container_id"] - console = Console() - with console.status("Tearing down..."): + with ui.status("Tearing down..."): # Stop any running deploy first, so the local controller and Redis # containers it spawned via docker-outside-of-docker get torn down # too. Removing the GC container itself doesn't touch them -- they're @@ -54,6 +52,6 @@ def run_quit(): os.remove(STATE_PATH) if already_gone: - print(f"Global Controller container {container_id[:12]} was already gone; cleaned up local state.") + ui.warn(f"Global Controller container {container_id[:12]} was already gone; cleaned up local state.") else: - print(f"Global Controller container {container_id[:12]} torn down (volume removed)") + ui.ok(f"Global Controller container {container_id[:12]} torn down (volume removed)") diff --git a/cli/canyonos/serve.py b/cli/canyonos/serve.py index 9515699..c96e336 100644 --- a/cli/canyonos/serve.py +++ b/cli/canyonos/serve.py @@ -1,18 +1,37 @@ """CLI output for the local dashboard stack.""" -from .dashboard_stack import run_dashboard +from canyonos import ui +from .dashboard_stack import ServeResult, run_dashboard -def run_serve() -> int: - def report(phase: str, message: str) -> None: - print(f"[serve] {phase}: {message}") +def serve_dashboard() -> ServeResult: + """Bring the dashboard up, reporting progress. Returns the stack's result.""" + # Phases drive the spinner while the stack comes up; the trace itself is + # only printed when something fails and the user needs to see how far it got. + trace = [] + + with ui.status("Starting the dashboard...") as spinner: + def report(phase: str, message: str) -> None: + trace.append((phase, message)) + spinner.update(message) + + result = run_dashboard(report) - result = run_dashboard(report) if result.ok: - print(f"Dashboard: {result.url}") - return 0 + return result - print(f"serve failed in {result.phase}: {result.message}") + for phase, message in trace: + ui.hint(f"{phase}: {message}") + ui.fail(f"serve failed in {result.phase}: {result.message}") if result.log_path: - print(f"log: {result.log_path}") - return 1 + ui.hint(f"log: {result.log_path}") + return result + + +def run_serve() -> int: + result = serve_dashboard() + if not result.ok: + return 1 + + ui.ok(f"Dashboard: {result.url}") + return 0 diff --git a/cli/canyonos/status.py b/cli/canyonos/status.py new file mode 100644 index 0000000..071297e --- /dev/null +++ b/cli/canyonos/status.py @@ -0,0 +1,55 @@ +""" +Logic for `canyonos status`: reports whether a deploy is currently running, +and if so, where the workflow (and, if up, the dashboard) answer. +""" + +from canyonos import ui +from canyonos.constants import WORKFLOW_ROUTE, default_config_path, workflow_api_port +from canyonos.dashboard_stack import _existing_dashboard_port +from canyonos.deploy import workflow_targets +from canyonos.gc import deploy_status, require_state + + +def run_status(): + state = require_state() + if state is None: + return + + status = deploy_status(state["port"]) + if not status or not status.get("running"): + ui.warn("No deploy is currently running.") + return + + ui.ok("Deploy is running.") + + # Same resolution `deploy` uses, so both report the address the container + # actually placed the workflow at and fall back to the configured api_port + # rather than a guess. + targets = workflow_targets(state["port"], workflow_api_port(default_config_path())) + for name, target_host, target_port in targets: + label = f"Workflow {name}" if name else "Workflow" + ui.say(f"{label}: {target_host}:{target_port}") + if not targets: + ui.hint("No workflow endpoints reported yet.") + + dashboard_port = _existing_dashboard_port() + if dashboard_port: + ui.say(f"Dashboard: 127.0.0.1:{dashboard_port}") + else: + ui.hint("Dashboard is not running. Run `canyonos serve` to start it.") + + if not targets: + return + + # The body is splatted into the workflow entrypoint as kwargs, so its keys + # are that function's parameter names -- `query` for every bundled example, + # but swap in whatever yours actually takes. + _, host, port = targets[0] + ui.blank() + ui.hint("Query the workflow:") + ui.say(f" curl -X POST http://{host}:{port}/{WORKFLOW_ROUTE} \\") + ui.say(' -H "Content-Type: application/json" \\') + ui.say(" -d '{\"query\": \"your question here\"}'") + ui.blank() + ui.hint("Check a request's result:") + ui.say(f" curl http://{host}:{port}/status/") diff --git a/cli/canyonos/stop.py b/cli/canyonos/stop.py index a3f96f3..519cf49 100644 --- a/cli/canyonos/stop.py +++ b/cli/canyonos/stop.py @@ -3,8 +3,7 @@ Controller container (SIGTERM, same teardown as Ctrl+C would trigger). """ -from rich.console import Console - +from canyonos import ui from canyonos.gc import GCError, post_clean, require_state @@ -13,10 +12,9 @@ def run_stop(): if state is None: return - console = Console() try: - with console.status("Stopping deploy..."): + with ui.status("Stopping deploy..."): post_clean(state["port"]) - print("Deploy stopped.") + ui.ok("Deploy stopped.") except GCError as e: - print(e) + ui.fail(e) diff --git a/cli/canyonos/sync.py b/cli/canyonos/sync.py index 20ac3c1..84f875c 100644 --- a/cli/canyonos/sync.py +++ b/cli/canyonos/sync.py @@ -13,6 +13,7 @@ import os import subprocess +from canyonos import ui from canyonos.gc import require_state from canyonos.init import GC_WORKSPACE_PATH @@ -27,14 +28,18 @@ def run_sync(): # Trailing "/." copies the *contents* of the current directory into # /workspace, rather than nesting it under /workspace/. src = os.path.join(os.getcwd(), ".") - print(f"Syncing {os.getcwd()} -> {container_id[:12]}:{GC_WORKSPACE_PATH} ...") - - result = subprocess.run( - ["docker", "cp", src, f"{container_id}:{GC_WORKSPACE_PATH}"] - ) + label = f"Syncing {os.getcwd()} -> {container_id[:12]}:{GC_WORKSPACE_PATH}" + + with ui.status(f"{label}..."): + # Captured so docker's own progress output doesn't clobber the spinner. + result = subprocess.run( + ["docker", "cp", src, f"{container_id}:{GC_WORKSPACE_PATH}"], + capture_output=True, + text=True, + ) if result.returncode != 0: - print("Sync failed.") + ui.fail(f"Sync failed: {result.stderr.strip() or result.stdout.strip()}") return False - print("Sync complete.") + ui.ok("Sync complete.") return True diff --git a/cli/canyonos/test.py b/cli/canyonos/test.py index 468527a..2409965 100644 --- a/cli/canyonos/test.py +++ b/cli/canyonos/test.py @@ -1,20 +1,28 @@ """ -Logic for `canyonos test`: smoke-test a project end to end on this machine. +Logic for `canyonos test`: check a project end to end on this machine. -Every agent's `provider` is rewritten to `local` for the duration of the run -(the original file is restored verbatim afterwards), the project is deployed -into the Global Controller container, one query is sent to the workflow's -`/main` endpoint, and its result -- or the error that came back -- is printed. +Four phases, each ending the run if it fails: the `.car/` artifact `canyonos +build` produced is verified statically, the project is deployed locally (every +agent's `provider` rewritten to `local` for the duration, the original file +restored verbatim afterwards), the running containers are checked against what +the config declared, and one prompt is sent to the workflow's `/main` endpoint. + +A passing run leaves nothing behind. A failing one leaves the Global Controller +container up, with the tail of its log, so there is something left to debug. """ import json import os +import socket +import subprocess import time import urllib.error import urllib.request -from rich.console import Console +from rich.panel import Panel +from rich.text import Text +from canyonos import ui from canyonos.constants import ( WORKFLOW_ROUTE, default_config_path, @@ -22,16 +30,25 @@ workflow_api_port, workspace_relative, ) +from canyonos.deploy import workflow_targets from canyonos.gc import GCError, deploy_status, post_deploy from canyonos.init import load_state, quit_existing, run_init from canyonos.sync import run_sync +from canyonos.theme import GREEN, WHITE +from canyonos.verify import ( + ARTIFACT_DIR, + VerificationError, + verify_build_artifact, + verify_runtime, +) DEFAULT_QUERY = "hello" # Generous: the first deploy of a project builds every agent image from scratch. READY_TIMEOUT = 900 REQUEST_TIMEOUT = 600 +SUBMIT_TIMEOUT = 30 POLL_INTERVAL = 2 - +LOG_TAIL_LINES = 40 def _force_local_providers(config_path): @@ -51,13 +68,21 @@ def _force_local_providers(config_path): return original -def _workflow_ready(api_port): +def _port_in_use(port): + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + return True + except OSError: + return False + + +def _workflow_ready(host, port): """True once the workflow's REST API answers at all. Any HTTP response counts -- /status/ 404s, which still proves the server is up and listening. """ - url = f"http://127.0.0.1:{api_port}/status/canyonos-test-probe" + url = f"http://{host}:{port}/status/canyonos-test-probe" try: urllib.request.urlopen(url, timeout=2) return True @@ -67,33 +92,32 @@ def _workflow_ready(api_port): return False -def _wait_for_workflow(gc_port, api_port, console): +def _wait_for_workflow(gc_port, api_port): deadline = time.time() + READY_TIMEOUT - with console.status("Building and starting containers..."): + with ui.status("Building images and starting containers..."): while time.time() < deadline: - if _workflow_ready(api_port): - return True + if _workflow_ready("127.0.0.1", api_port): + return if not (deploy_status(gc_port) or {}).get("running", False): - return False + raise _TestFailed("The deploy stopped before the workflow came up.") time.sleep(POLL_INTERVAL) - print(f"Timed out after {READY_TIMEOUT}s waiting for the workflow to come up.") - return False + raise _TestFailed(f"Timed out after {READY_TIMEOUT}s waiting for the workflow to come up.") -def _send_query(api_port, query): - url = f"http://127.0.0.1:{api_port}/{WORKFLOW_ROUTE}" +def _send_query(host, port, query): + url = f"http://{host}:{port}/{WORKFLOW_ROUTE}" body = json.dumps({"query": query}).encode() req = urllib.request.Request( url, data=body, headers={"Content-Type": "application/json"}, method="POST" ) - with urllib.request.urlopen(req, timeout=30) as resp: + with urllib.request.urlopen(req, timeout=SUBMIT_TIMEOUT) as resp: return json.loads(resp.read())["request_id"] -def _await_result(api_port, request_id, console): - url = f"http://127.0.0.1:{api_port}/status/{request_id}" +def _await_result(host, port, request_id): + url = f"http://{host}:{port}/status/{request_id}" deadline = time.time() + REQUEST_TIMEOUT - with console.status("Running query..."): + with ui.status("Running query..."): while time.time() < deadline: try: with urllib.request.urlopen(url, timeout=10) as resp: @@ -107,67 +131,266 @@ def _await_result(api_port, request_id, console): return {"status": "timeout"} -def run_test(config_path=None, query=None): - console = Console() - config_path = config_path or default_config_path() - query = query or DEFAULT_QUERY +def _log_tail(container_id): + result = subprocess.run( + ["docker", "logs", "--tail", str(LOG_TAIL_LINES), container_id], + capture_output=True, + text=True, + ) + return (result.stdout + result.stderr).strip() or None - config_path = workspace_relative(config_path) - if config_path is None: - print("Config must be inside the project directory being synced.") - return 1 - if not os.path.isfile(config_path): - print(f"Config file not found: {config_path}. Run `canyonos build` first.") - return 1 +class _TestFailed(Exception): + """Ends the run early, carrying a message fit for either output mode.""" - api_port = workflow_api_port(config_path) - if api_port is None: - print(f"No agent with `type: workflow` in {config_path}; nothing to test.") - return 1 - print(f"Testing {config_path} locally (query: {query!r})") - original_config = _force_local_providers(config_path) +class _Run: + """One `canyonos test` invocation: the phases it got through, and what they found.""" + + def __init__(self, query): + self.query = query + self.started = time.monotonic() + # Only once a deploy is under way is the container worth keeping and its + # log worth reading; before that it holds nothing about the failure. + self.deploy_started = False + self.phases = [] + self.validation = None + self.runtime = None + self.endpoint = None + self.result = None + self.error = None + self.log_tail = None + + def begin(self, name, number, title): + """Open a phase, recorded as failed until `done` says otherwise.""" + self.phases.append({"name": name, "ok": False, "detail": None}) + ui.blank() + ui.say(f"[{number}/4] {title}") + + def done(self, detail=None): + self.phases[-1].update(ok=True, detail=detail) + + def failed(self, detail): + if self.phases: + self.phases[-1]["detail"] = detail + + def elapsed(self): + return round(time.monotonic() - self.started, 3) + + +def _verify_build(run, config_path): + run.begin("verify_build", 1, "Verify build artifact") + + # A project ported before the .car layout keeps its config at the top level; + # there is no build artifact to check, so the deploy phases still run. + if not config_path.startswith(f"{ARTIFACT_DIR}{os.sep}"): + ui.warn(f"No `{ARTIFACT_DIR}/` artifact -- deploying {config_path} as it is.") + ui.hint(" -> `canyonos build` produces one, and gives this phase something to check.") + run.done("skipped: no .car/ artifact") + return try: - run_init() - if not run_sync(): - return 1 + run.validation = verify_build_artifact() + except VerificationError as e: + raise _TestFailed(str(e)) from None + stale = len(run.validation["stale"]) + run.done(f"{run.validation['warnings']} warning(s), {stale} stale source(s)") + + +def _deploy_locally(run, config_path, api_port): + run.begin("deploy", 2, "Deploy locally") + run_init(banner=False) + + if not run_sync(): + raise _TestFailed("Could not sync the project into the container.") + + # Only the gRPC host port is bumped when a port is taken (the local runtime's + # launch retry), so an occupied api_port dies 50 attempts later as "no free + # port found". `canyonos serve` also starts looking for its web port at 8080. + if _port_in_use(api_port): + raise _TestFailed( + f"Port {api_port} is already in use, and the workflow needs it. Free it " + f"(`canyonos quit` stops a previous deploy) or change `api_port` in {config_path}." + ) + + state = load_state() + try: + post_deploy(state["port"], config_path) + except GCError as e: + raise _TestFailed(str(e)) from None + run.deploy_started = True - state = load_state() - try: - post_deploy(state["port"], config_path) - except GCError as e: - print(e) - return 1 + _wait_for_workflow(state["port"], api_port) + run.done(f"Global Controller on port {state['port']}") + return state - if not _wait_for_workflow(state["port"], api_port, console): - print("The deploy did not come up. Run `canyonos logs` to see why.") - return 1 - try: - request_id = _send_query(api_port, query) - except OSError as e: - print(f"Could not reach the workflow on port {api_port}: {e}") - return 1 - result = _await_result(api_port, request_id, console) - except KeyboardInterrupt: - print("\nTest cancelled.") - return 1 +def _verify_runtime(run, config_path, gc_port): + run.begin("verify_runtime", 3, "Verify runtime") + try: + run.runtime = verify_runtime(config_path, gc_port) + except VerificationError as e: + raise _TestFailed(str(e)) from None + run.done(f"{len(run.runtime['agents'])} agent(s) up") + + +def _query(run, gc_port, api_port): + run.begin("query", 4, "Query the workflow") + targets = workflow_targets(gc_port, api_port) + if not targets: + raise _TestFailed("The deploy reported no workflow endpoint to query.") + + _, host, port = targets[0] + run.endpoint = f"http://{host}:{port}/{WORKFLOW_ROUTE}" + ui.say(f"POST {run.endpoint} {json.dumps({'query': run.query})}") + + try: + request_id = _send_query(host, port, run.query) + except OSError as e: + raise _TestFailed(f"Could not reach the workflow at {run.endpoint}: {e}") from None + + data = _await_result(host, port, request_id) + status = data.get("status") + if status == "error": + raise _TestFailed(data.get("error") or "the workflow returned an error.") + if status != "done": + raise _TestFailed(f"The workflow did not finish within {REQUEST_TIMEOUT}s.") + + run.result = data.get("result") + run.done(f"answered in {run.elapsed()}s") + + +def _run_test(run): + """Walk the four phases, restoring the config whatever happens.""" + config_path = workspace_relative(default_config_path()) + if config_path is None: + raise _TestFailed("Config must be inside the project directory being synced.") + if not os.path.isfile(config_path): + raise _TestFailed(f"No config at {config_path}. Run `canyonos build` first.") + + _verify_build(run, config_path) + + api_port = workflow_api_port(config_path) + if api_port is None: + raise _TestFailed(f"No agent with `type: workflow` in {config_path}; nothing to test.") + + original_config = _force_local_providers(config_path) + try: + state = _deploy_locally(run, config_path, api_port) + _verify_runtime(run, config_path, state["port"]) + _query(run, state["port"], api_port) finally: with open(config_path, "w") as f: f.write(original_config) - # A smoke test leaves nothing behind: run_init() started this container. - quit_existing() - status = result.get("status") - if status == "done": - print("Test passed.") - print(json.dumps(result.get("result"), indent=2)) - return 0 - if status == "error": - print(f"Test failed: {result.get('error')}") +# ------------------------------------------------------------------ # +# Output # +# ------------------------------------------------------------------ # + + +def _summary_body(run): + body = Text() + body.append("Query ", "dim") + body.append(run.query, WHITE) + if run.endpoint: + body.append("\nEndpoint ", "dim") + body.append(run.endpoint, WHITE) + body.append("\nElapsed ", "dim") + body.append(f"{run.elapsed()}s", WHITE) + + body.append("\n") + for phase in run.phases: + body.append("\n") + body.append("✓ " if phase["ok"] else "✗ ", GREEN if phase["ok"] else "bold red") + body.append(f"{phase['name']:<16}", WHITE) + # The failing phase's detail is the error, spelled out below in full. + body.append(phase["detail"] if phase["ok"] else "", "dim") + + body.append("\n\n") + if run.error is None: + body.append("Result ", "dim") + body.append(json.dumps(run.result, indent=2), WHITE) else: - print(f"Test failed: workflow did not finish within {REQUEST_TIMEOUT}s.") - return 1 + body.append(run.error, "bold red") + return body + + +def _print_summary(run): + passed = run.error is None + ui.blank() + ui.panel( + Panel( + _summary_body(run), + title=f"[bold {GREEN}]Test passed[/]" if passed else "[bold red]Test failed[/]", + title_align="left", + border_style=GREEN if passed else "red", + padding=(1, 4), + ) + ) + ui.blank() + + +def _print_failure_logs(run): + if run.log_tail: + ui.hint(f"last {LOG_TAIL_LINES} lines of the Global Controller log:") + ui.say(run.log_tail) + ui.blank() + ui.hint("Containers left running for inspection: `canyonos logs` | `canyonos quit`") + + +def _payload(run): + return { + "ok": run.error is None, + "query": run.query, + "elapsed_s": run.elapsed(), + "phases": run.phases, + "validation": run.validation, + "runtime": run.runtime, + "result": run.result, + "error": run.error, + "log_tail": run.log_tail, + } + + +def run_test(prompt=None, as_json=False): + run = _Run(prompt or DEFAULT_QUERY) + ui.set_quiet(as_json) + + try: + container_live = False + try: + _run_test(run) + except _TestFailed as e: + run.error = str(e) + except KeyboardInterrupt: + run.error = "cancelled by user" + except RuntimeError as e: + # Docker unreachable, image pull failed, no free port: all carry a + # readable message, and `--json` needs it inside the payload. + run.error = str(e) + + if run.error is not None: + run.failed(run.error) + + if run.error is not None and run.deploy_started: + # Read the log before anything else touches the container, and leave + # it running -- a torn-down deploy can't be diagnosed. + try: + run.log_tail = _log_tail(load_state()["container_id"]) + container_live = True + except (FileNotFoundError, OSError): + pass + else: + quit_existing() + + if as_json: + print(json.dumps(_payload(run), indent=2)) + else: + _print_summary(run) + if container_live: + _print_failure_logs(run) + + return 0 if run.error is None else 1 + finally: + ui.set_quiet(False) diff --git a/cli/canyonos/ui.py b/cli/canyonos/ui.py new file mode 100644 index 0000000..056999a --- /dev/null +++ b/cli/canyonos/ui.py @@ -0,0 +1,67 @@ +""" +The CLI's one output surface: every user-facing line goes through here so the +whole tool speaks with the same palette, symbols and spinner. + +Messages are emitted as literal text, never as rich markup, so a path or an +error containing square brackets can't be swallowed as a style tag. +""" + +from contextlib import contextmanager + +from rich.console import Console +from rich.text import Text + +from canyonos.theme import GRADIENT, GREEN, WHITE + +console = Console() + + +def set_quiet(quiet): + """Silence every helper here, so `canyonos test --json` emits only its payload.""" + console.quiet = quiet + + +def _emit(message, style, symbol=None): + parts = [(f"{symbol} ", style)] if symbol else [] + parts.append((str(message), WHITE if symbol else style)) + console.print(Text.assemble(*parts)) + + +def say(message): + _emit(message, WHITE) + + +def ok(message): + _emit(message, GREEN, "✓") + + +def fail(message): + _emit(message, "bold red", "✗") + + +def warn(message): + _emit(message, "yellow", "!") + + +def hint(message): + _emit(message, "dim") + + +def blank(): + console.print() + + +def gradient(text): + """Print `text` line by line down the brand ramp (the `init` banner).""" + for line, color in zip(text.splitlines(), GRADIENT): + console.print(line, style=color) + + +def panel(renderable): + console.print(renderable) + + +@contextmanager +def status(message): + with console.status(message) as spinner: + yield spinner diff --git a/cli/canyonos/verify.py b/cli/canyonos/verify.py new file mode 100644 index 0000000..d047885 --- /dev/null +++ b/cli/canyonos/verify.py @@ -0,0 +1,291 @@ +""" +The two verification passes behind `canyonos test`. + +`verify_build_artifact` checks the `.car/` tree a `canyonos build` produced, +before any container is started: the layout, the porting skill's own validator, +and whether the sources have moved on since the port was taken. + +`verify_runtime` checks a running local deploy against what the config declared +-- every image built, every replica up -- because the controller logs a warning +and carries on when an agent never becomes healthy, so a workflow that answers +is not on its own proof that the deploy is complete. +""" + +import hashlib +import json +import os +import subprocess +import sys + +import yaml +from rich.table import Table + +from canyonos import gc, ui +from canyonos.build import AGENTS, install_skill +from canyonos.constants import DEFAULT_API_PORT +from canyonos.init import STATE_DIR +from canyonos.theme import GREEN + +ARTIFACT_DIR = ".car" +SOURCE_DIR = "app" +CONFIG_REL = "config/global_controller.yaml" +PORTING_STATE_REL = "config/.porting-state.json" + +VALIDATOR_NAME = "validate.py" +SKILL_CACHE_DIR = os.path.join(STATE_DIR, "skill") + +# These two rules decide their verdict by importing `ventis` and probing it for +# env-file injection and editable-install support. The runtime lives in the +# Global Controller image, not on the host running this CLI, so the probe always +# comes back empty here and the rules report a failure that isn't one. +CAPABILITY_GATED_CHECKS = frozenset({"V030", "V031"}) + +RUNTIME_PREFIX = "ventis-local-" + + +class VerificationError(Exception): + """A check that should end the run, carrying a message fit to print.""" + + +# ------------------------------------------------------------------ # +# Build artifact # +# ------------------------------------------------------------------ # + + +def _find_validator(project_root): + """Path to the porting skill's validate.py, fetching the skill if needed.""" + for spec in AGENTS.values(): + for skill_dir in spec["skill_dirs"].values(): + if not os.path.isabs(skill_dir): + skill_dir = os.path.join(project_root, skill_dir) + candidate = os.path.join(skill_dir, VALIDATOR_NAME) + if os.path.isfile(candidate): + return candidate + + cached = os.path.join(SKILL_CACHE_DIR, VALIDATOR_NAME) + if os.path.isfile(cached): + return cached + if install_skill(SKILL_CACHE_DIR) and os.path.isfile(cached): + return cached + return None + + +def _run_validator(validator, artifact_dir): + """The validator's parsed --json report, or None if it produced no report.""" + result = subprocess.run( + [sys.executable, validator, artifact_dir, "-c", CONFIG_REL, "--json"], + capture_output=True, + text=True, + ) + try: + return json.loads(result.stdout) + except ValueError: + detail = (result.stderr or result.stdout).strip().splitlines() + ui.warn(f" The porting validator did not run: {detail[-1] if detail else 'no output'}") + return None + + +def _drop_unprobeable(report): + """Remove the rules that can only be judged with `ventis` importable. + + Their verdict without it is not merely uncertain, it is wrong: V030 reports + that the runtime never reads `env_file` when the container's runtime does. + """ + if report.get("capabilities", {}).get("ventis"): + return 0 + + kept = [] + dropped = 0 + for finding in report.get("findings") or []: + if finding["check"] in CAPABILITY_GATED_CHECKS: + if finding["level"] == "ERROR": + report["errors"] = max(report.get("errors", 0) - 1, 0) + elif finding["level"] == "WARN": + report["warnings"] = max(report.get("warnings", 0) - 1, 0) + dropped += 1 + continue + kept.append(finding) + report["findings"] = kept + return dropped + + +_LEVEL_EMITTER = {"ERROR": ui.fail, "WARN": ui.warn} + + +def _report_findings(findings): + for finding in sorted(findings, key=lambda f: (f["level"] != "ERROR", f["check"])): + where = finding.get("path") or "" + if where and finding.get("line"): + where = f"{where}:{finding['line']}" + parts = [finding["check"], where, finding["summary"]] + line = " ".join(part for part in parts if part) + _LEVEL_EMITTER.get(finding["level"], ui.hint)(f" {line}") + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(65536), b""): + digest.update(block) + return digest.hexdigest() + + +def _stale_sources(project_root, artifact_dir): + """Recorded sources that changed or vanished since the port was taken.""" + try: + with open(os.path.join(artifact_dir, PORTING_STATE_REL)) as f: + state = json.load(f) + except (OSError, ValueError): + return [] + + stale = [] + for relative, expected in (state.get("source_files") or {}).items(): + # The skill's own files are recorded alongside the project's; a newer + # skill would otherwise read as the application having changed. + if relative.startswith(".claude/"): + continue + path = os.path.join(project_root, relative) + if not os.path.isfile(path) or _sha256(path) != expected: + stale.append(relative) + return sorted(stale) + + +def verify_build_artifact(project_root="."): + """Check the `.car/` tree. Raises VerificationError if it can't be deployed.""" + artifact_dir = os.path.join(project_root, ARTIFACT_DIR) + config_path = os.path.join(artifact_dir, CONFIG_REL) + + if not os.path.isfile(config_path) or not os.path.isdir( + os.path.join(artifact_dir, SOURCE_DIR) + ): + raise VerificationError( + f"No `{ARTIFACT_DIR}/` artifact here (expected {CONFIG_REL} beside " + f"{SOURCE_DIR}/). Run `canyonos build` first." + ) + ui.ok(f"{ARTIFACT_DIR}/ layout (config/ + {SOURCE_DIR}/)") + + summary = {"errors": 0, "warnings": 0, "findings": [], "stale": []} + + validator = _find_validator(project_root) + if validator is None: + ui.warn("Could not fetch the porting validator; skipping artifact checks.") + ui.hint(" The deploy below still runs -- `canyonos doctor` checks the fetch path.") + else: + report = _run_validator(validator, os.path.abspath(artifact_dir)) + if report is not None: + skipped = _drop_unprobeable(report) + summary.update( + errors=report.get("errors", 0), + warnings=report.get("warnings", 0), + findings=report.get("findings", []), + ) + counts = f"{summary['errors']} error(s), {summary['warnings']} warning(s)" + (ui.fail if summary["errors"] else ui.ok)(f"porting validator: {counts}") + _report_findings(summary["findings"]) + if skipped: + ui.hint(f" {skipped} rule(s) need the ventis runtime to judge and were skipped") + + summary["stale"] = _stale_sources(project_root, artifact_dir) + for relative in summary["stale"]: + ui.warn(f" source changed since the port: {relative}") + if summary["stale"]: + ui.hint(" -> re-run `canyonos build` to bring the artifact back in step") + + if summary["errors"]: + raise VerificationError( + f"The build artifact has {summary['errors']} validation error(s); fix them " + "or re-run `canyonos build`." + ) + return summary + + +# ------------------------------------------------------------------ # +# Runtime # +# ------------------------------------------------------------------ # + + +def _built_images(): + result = subprocess.run( + ["docker", "images", "--format", "{{.Repository}}"], capture_output=True, text=True + ) + return set(result.stdout.split()) + + +def _running_containers(): + result = subprocess.run( + ["docker", "ps", "--filter", f"name={RUNTIME_PREFIX}", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + return result.stdout.split() + + +def _runtime_table(rows): + table = Table(border_style=GREEN, header_style=f"bold {GREEN}", title_style=f"bold {GREEN}") + for column in ("Agent", "Image", "Replicas", "Endpoint"): + table.add_column(column) + for row in rows: + replicas = f"{row['running']}/{row['expected']}" + style = "" if row["ok"] else "bold red" + table.add_row( + row["name"], + row["image"] if row["image_built"] else f"{row['image']} (missing)", + replicas, + row["endpoint"] or "-", + style=style, + ) + return table + + +def verify_runtime(config_path, gc_port): + """Check the running deploy against the config. Raises VerificationError on a gap.""" + with open(config_path) as f: + config = yaml.safe_load(f) or {} + + images = _built_images() + containers = _running_containers() + endpoints = { + endpoint.get("name"): f"{endpoint['host']}:{endpoint['port']}" + for endpoint in gc.workflow_endpoints(gc_port) + if endpoint.get("host") and endpoint.get("port") + } + + rows = [] + problems = [] + for agent in config.get("agents") or []: + name = agent.get("name") + if not name: + continue + # Image and container names the local provider derives from the agent name. + image = f"ventis-{name.lower()}" + expected = int(agent.get("replicas", 1) or 1) + running = sum(1 for c in containers if c.startswith(f"{RUNTIME_PREFIX}{name.lower()}-")) + image_built = image in images + + if not image_built: + problems.append(f"{name}: image {image} was never built") + elif running < expected: + problems.append(f"{name}: {running} of {expected} replicas running") + + endpoint = endpoints.get(name) + if endpoint is None and agent.get("type") == "workflow": + # The container only reports endpoints it has instance records for; + # locally the published port is the one the config asked for. + endpoint = f"127.0.0.1:{agent.get('api_port', DEFAULT_API_PORT)}" + + rows.append( + { + "name": name, + "image": image, + "image_built": image_built, + "expected": expected, + "running": running, + "endpoint": endpoint, + "ok": image_built and running >= expected, + } + ) + + ui.panel(_runtime_table(rows)) + if problems: + raise VerificationError("The deploy is incomplete -- " + "; ".join(problems)) + return {"agents": rows} diff --git a/cli/cli.py b/cli/cli.py index 770741f..a6bd390 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -7,8 +7,8 @@ import importlib.metadata import sys +from canyonos import ui from canyonos.clean import run_clean -from canyonos.constants import default_config_path from canyonos.config import run_config from canyonos.deploy import run_deploy from canyonos.build import run_build @@ -17,49 +17,11 @@ from canyonos.new_app import run_new_app from canyonos.quit import run_quit from canyonos.serve import run_serve +from canyonos.status import run_status from canyonos.stop import run_stop from canyonos.test import DEFAULT_QUERY, run_test from utils.help_screen import DESCRIPTIONS, print_custom_help -def cmd_quit(args): - run_quit() - -def cmd_new_app(args): - # Note, not tested much, keeping this in the back burner for now while we flesh out the main path - run_new_app() - -def cmd_deploy(args): - run_deploy(args.config, serve=args.serve) - -def cmd_clean(args): - run_clean() - -def cmd_stop(args): - run_stop() - -def cmd_logs(args): - run_logs() - -def cmd_config(args): - run_config() - -def cmd_build(args): - run_build() - -def cmd_doctor(args): - sys.exit(0 if run_doctor() else 1) - -def cmd_serve(args): - sys.exit(run_serve()) - -def cmd_test(args): - sys.exit(run_test(args.config, query=args.query)) - - -def cmd_version(args): - print(f"canyonos {importlib.metadata.version('canyonos')}") - - def _parse_bool(value): if value.lower() in ("true", "1", "yes"): return True @@ -80,14 +42,16 @@ def main(): # Subparsers keep the stock argparse help, so `canyonos -h` still # describes that command instead of reprinting the top-level screen. subparsers = parser.add_subparsers(dest="command", parser_class=argparse.ArgumentParser) - config_default = default_config_path() - def add(name): + def add(name, run): # A KeyError here means the command has no entry on the help screen. - return subparsers.add_parser(name, help=DESCRIPTIONS[name]) + command = subparsers.add_parser(name, help=DESCRIPTIONS[name]) + command.set_defaults(func=run) + return command - add("new-app").set_defaults(func=cmd_new_app) - deploy = add("deploy") + # Note, not tested much, keeping this in the back burner for now while we flesh out the main path + add("new-app", lambda args: run_new_app()) + deploy = add("deploy", lambda args: run_deploy(args.config, serve=args.serve, verbose=args.verbose)) deploy.add_argument( "-c", "--config", @@ -100,30 +64,34 @@ def add(name): metavar="true|false", help="Automatically launch the local dashboard (canyonos serve) once the workflow is up (default: true)", ) - deploy.set_defaults(func=cmd_deploy) - add("clean").set_defaults(func=cmd_clean) - add("stop").set_defaults(func=cmd_stop) - add("logs").set_defaults(func=cmd_logs) - add("quit").set_defaults(func=cmd_quit) - add("config").set_defaults(func=cmd_config) - add("build").set_defaults(func=cmd_build) - add("doctor").set_defaults(func=cmd_doctor) - add("version").set_defaults(func=cmd_version) - add("serve").set_defaults(func=cmd_serve) - test = add("test") - test.add_argument( - "-c", - "--config", - default=config_default, - help=f"Path to global controller config (default: {config_default})", + deploy.add_argument( + "-v", + "--verbose", + action="store_true", + help="Stream the container's full build and deploy logs instead of a progress summary", ) + add("clean", lambda args: run_clean()) + add("stop", lambda args: run_stop()) + add("logs", lambda args: run_logs()) + add("quit", lambda args: run_quit()) + add("config", lambda args: run_config()) + add("build", lambda args: run_build()) + add("doctor", lambda args: sys.exit(0 if run_doctor() else 1)) + add("version", lambda args: ui.say(f"canyonos {importlib.metadata.version('canyonos')}")) + add("serve", lambda args: sys.exit(run_serve())) + add("status", lambda args: run_status()) + test = add("test", lambda args: sys.exit(run_test(args.prompt, as_json=args.json))) test.add_argument( - "-q", - "--query", + "prompt", + nargs="?", default=DEFAULT_QUERY, - help=f"Query sent to the workflow (default: {DEFAULT_QUERY!r})", + help=f"Prompt sent to the workflow (default: {DEFAULT_QUERY!r})", + ) + test.add_argument( + "--json", + action="store_true", + help="Print a single JSON result object and nothing else (for CI)", ) - test.set_defaults(func=cmd_test) args = parser.parse_args() if not getattr(args, "command", None): @@ -135,7 +103,7 @@ def add(name): except RuntimeError as e: # Docker unreachable, image pull failed, no free port -- all already # carry a readable message, so print it rather than a traceback. - print(e) + ui.fail(e) sys.exit(1) diff --git a/cli/pyproject.toml b/cli/pyproject.toml index e85c825..f11a241 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "canyonos" -version = "0.1.4" +version = "0.1.5" description = "CanyonOS CLI" requires-python = ">=3.10" dependencies = [ diff --git a/cli/utils/help_screen.py b/cli/utils/help_screen.py index 94b94a0..81ea3ca 100644 --- a/cli/utils/help_screen.py +++ b/cli/utils/help_screen.py @@ -1,38 +1,50 @@ """Custom help screen for the canyonos CLI.""" -from rich.console import Console from rich.panel import Panel from rich.table import Table from rich.text import Text +from canyonos import ui +from canyonos.theme import GREEN, WHITE + # The single source of truth for command descriptions: cli.py registers every # subparser through this table, so a command can't be added to one and missed # in the other. CORE_COMMANDS = ( - ("build", "Build a compatable workflow with an agent"), - ("deploy", "Deploy agents"), + ("build", "Port an existing project into a CanyonOS workflow"), + ("deploy", "Build and launch the workflow, then open the dashboard"), ("config", "Configure project settings"), ) +# The three teardown commands differ only in what they leave behind, so each +# description says so explicitly rather than all three reading as "stop stuff". UTIL_COMMANDS = ( - ("clean", "Remove the generated .car folder from build"), - ("doctor", "See if all required tools are up"), - ("logs", "View canyonos logs"), + ("clean", "Delete the generated .car folder from this project"), + ("doctor", "Check Docker, git and a coding agent are all available"), + ("logs", "Follow the running deploy's logs"), ("new-app", "Create a barebones CanyonOS project"), - ("quit", "Shut down CanyonOS services"), + ("quit", "Stop the deploy and remove the container and its files"), ("serve", "Start local CanyonOS dashboard"), - ("stop", "Stop running containers"), - ("test", "Run the deployed workflow locally with a test query"), + ("status", "Check whether a deploy is running and where it answers"), + ("stop", "Stop the running deploy, keeping the container and files"), + ("test", "Deploy locally and run one prompt end to end"), ("version", "Print canyonos version"), ) DESCRIPTIONS = dict(CORE_COMMANDS + UTIL_COMMANDS) +# Both columns are sized from the widest entry across BOTH tables, so Core and +# Utils line up with each other instead of each shrinking to fit its own rows. +_ALL_COMMANDS = CORE_COMMANDS + UTIL_COMMANDS +_NAME_WIDTH = max(len(name) for name, _ in _ALL_COMMANDS) +_DESCRIPTION_WIDTH = max(len(description) for _, description in _ALL_COMMANDS) + + def _command_table(commands): table = Table(show_header=False, border_style="dim", padding=(0, 2)) - table.add_column(style="cyan", width=20) - table.add_column(style="white") + table.add_column(style=f"bold {GREEN}", width=_NAME_WIDTH) + table.add_column(style=WHITE, width=_DESCRIPTION_WIDTH) for name, description in commands: table.add_row(name, description) return table @@ -40,21 +52,19 @@ def _command_table(commands): def print_custom_help(): """Print a custom, visually appealing help screen.""" - console = Console() - - title = Text("CanyonOS CLI", style="bold cyan") - subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease\n", style="dim") - console.print(Panel(title + subtitle, border_style="cyan", padding=(1, 2))) + title = Text("CanyonOS CLI", style=f"bold {GREEN}") + subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease", style="dim") + ui.console.print(Panel(title + subtitle, border_style=GREEN, padding=(1, 2))) - console.print("\n[bold yellow]Core Commands[/bold yellow]") - console.print(_command_table(CORE_COMMANDS)) + ui.console.print(f"\n[bold {GREEN}]Core Commands[/]") + ui.console.print(_command_table(CORE_COMMANDS)) - console.print("\n[bold yellow]Utils[/bold yellow]") - console.print(_command_table(UTIL_COMMANDS)) + ui.console.print(f"\n[bold {GREEN}]Utils[/]") + ui.console.print(_command_table(UTIL_COMMANDS)) - console.print("\n[bold green]Quick Start:[/bold green]") - console.print(" [dim]1.[/dim] cd [cyan]into-your-workflow-root-dir[/cyan]") - console.print(" [dim]2.[/dim] canyonos build (opens coding agent)") - console.print(" [dim]3.[/dim] canyonos deploy (UI automatically starts)") + ui.console.print(f"\n[bold {GREEN}]Quick Start:[/]") + ui.console.print(f" [dim]1.[/dim] cd [{GREEN}]into-your-workflow-root-dir[/]") + ui.console.print(" [dim]2.[/dim] canyonos build (opens coding agent)") + ui.console.print(" [dim]3.[/dim] canyonos deploy (UI automatically starts)") - console.print("[dim]For command-specific help: [cyan]canyonos --help[/cyan][/dim]\n") + ui.console.print(f"[dim]For command-specific help: [{GREEN}]canyonos --help[/][/dim]\n") diff --git a/cli/utils/tui.py b/cli/utils/tui.py index 2f0f15d..3154062 100644 --- a/cli/utils/tui.py +++ b/cli/utils/tui.py @@ -8,12 +8,18 @@ import termios import tty +from canyonos.theme import GREEN + UP_KEYS = ("\x1b[A", "\x1bOA", "k") DOWN_KEYS = ("\x1b[B", "\x1bOB", "j") CANCEL_KEYS = ("\x03", "\x1b") DELETE_KEYS = ("d", "D") QUIT_KEYS = ("q", "Q") +# The brand green as a raw truecolor escape: this menu writes ANSI directly +# rather than going through rich, but shares the CLI's one palette. +_GREEN = "\x1b[38;2;{};{};{}m".format(*(int(GREEN[i:i + 2], 16) for i in (1, 3, 5))) + # Sentinel returned (paired with the hovered value) when the delete key is # pressed and `deletable=True`. Callers check `result[0] is DELETE_ACTION`. DELETE_ACTION = object() @@ -65,7 +71,7 @@ def select_menu(options, title, deletable=False, quittable=False): def frame(): lines = [f"\x1b[1m{title}\x1b[0m", ""] for i, (_, label) in enumerate(options): - lines.append(f"\x1b[36m❯ {label}\x1b[0m" if i == idx else f" {label}") + lines.append(f"{_GREEN}❯ {label}\x1b[0m" if i == idx else f" {label}") hint = "↑/↓ move · 1-9 jump · enter select" if deletable: hint += " · d delete" diff --git a/examples/finance/agents/finance_agent.py b/examples/finance/agents/finance_agent.py index 7c36ef5..db70b01 100644 --- a/examples/finance/agents/finance_agent.py +++ b/examples/finance/agents/finance_agent.py @@ -1,4 +1,11 @@ -from vllm_agent import VllmAgent +# `agents.vllm_agent` is where the generated VllmAgent stub actually lands +# inside this agent's own Docker container (stubs are copied to their source +# agent's own entrypoint-mirrored path -- see ventis/stub_generator.py). The +# bare `vllm_agent` fallback covers running outside that layout. +try: + from agents.vllm_agent import VllmAgent +except ImportError: + from vllm_agent import VllmAgent # Example of a simple finance agent diff --git a/examples/finance/workflow/example_workflow.py b/examples/finance/workflow/example_workflow.py index e4b6ce5..644e483 100644 --- a/examples/finance/workflow/example_workflow.py +++ b/examples/finance/workflow/example_workflow.py @@ -4,7 +4,7 @@ # Start agents first: python src/controller/global_controller.py # Then run this file: python examples/workflow.py # Test: -# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' -d '{"ticker": "AAPL"}' +# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' -d '{"query": "AAPL"}' # curl http://localhost:8080/status/ import sys @@ -22,13 +22,13 @@ from agents.market_agent import MarketResearchAgent -def main(ticker: str = "AAPL"): +def main(query: str = "AAPL"): finance = FinanceAgent() market = MarketResearchAgent() # Call finance agent functions - price = finance.get_stock_price(ticker=ticker) - company = finance.get_company_name(ticker=ticker) + price = finance.get_stock_price(ticker=query) + company = finance.get_company_name(ticker=query) # Call market agent functions trend = market.get_market_trend(sector="tech") diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md index a182896..38483be 100644 --- a/examples/helloworld/README.md +++ b/examples/helloworld/README.md @@ -14,7 +14,7 @@ ventis deploy # Test with curl curl -X POST http://:8080/main \ -H 'Content-Type: application/json' \ - -d '{"name": "World"}' + -d '{"query": "World"}' # Check result curl http://:8080/status/ @@ -52,5 +52,5 @@ Pass `_context` in your curl request to set the caller identity: ```bash curl -X POST http://localhost:8080/main \ -H 'Content-Type: application/json' \ - -d '{"name": "World", "_context": {"origin": "admin"}}' + -d '{"query": "World", "_context": {"origin": "admin"}}' ``` diff --git a/examples/helloworld/workflow/example_workflow.py b/examples/helloworld/workflow/example_workflow.py index 6bafff3..693590e 100644 --- a/examples/helloworld/workflow/example_workflow.py +++ b/examples/helloworld/workflow/example_workflow.py @@ -2,7 +2,7 @@ # This file demonstrates how to call agent stubs and deploy as a REST API. # # After running `ventis build` and `ventis deploy`: -# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' -d '{"name": "World"}' +# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' -d '{"query": "World"}' # curl http://localhost:8080/status/ import sys @@ -18,9 +18,9 @@ from agents.example_agent import ExampleAgent -def main(name: str = "World"): +def main(query: str = "World"): agent = ExampleAgent() - greeting = agent.hello(name=name) + greeting = agent.hello(name=query) return {"greeting": greeting.value()} diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 28069ac..253a2d2 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,14 +8,18 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. -import os -import sys - import json import math -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -from price_agent import PriceAgent +# `agents.price_agent` is where the generated PriceAgent stub actually lands +# inside this agent's own Docker container (stubs are copied to their source +# agent's own entrypoint-mirrored path -- see ventis/stub_generator.py). The +# bare `price_agent` fallback covers running outside that layout (e.g. local +# dev, where `ventis build` only emits a flat stubs/ directory). +try: + from agents.price_agent import PriceAgent +except ImportError: + from price_agent import PriceAgent TRADING_DAYS = 252 diff --git a/examples/text2sql/agents/sql_generator_agent.py b/examples/text2sql/agents/sql_generator_agent.py index 6320089..4964ce6 100644 --- a/examples/text2sql/agents/sql_generator_agent.py +++ b/examples/text2sql/agents/sql_generator_agent.py @@ -9,7 +9,14 @@ # These calls sit on the request's critical path, so the scheduler should # prioritize them over background work. -from vllm_agent import VllmAgent +# `agents.vllm_agent` is where the generated VllmAgent stub actually lands +# inside this agent's own Docker container (stubs are copied to their source +# agent's own entrypoint-mirrored path -- see ventis/stub_generator.py). The +# bare `vllm_agent` fallback covers running outside that layout. +try: + from agents.vllm_agent import VllmAgent +except ImportError: + from vllm_agent import VllmAgent class SQLGeneratorAgent(object): diff --git a/examples/text2sql/workflow/text2sql_workflow.py b/examples/text2sql/workflow/text2sql_workflow.py index ac9801d..b8ce45b 100644 --- a/examples/text2sql/workflow/text2sql_workflow.py +++ b/examples/text2sql/workflow/text2sql_workflow.py @@ -11,7 +11,7 @@ # Test: # curl -X POST http://localhost:8080/main \ # -H 'Content-Type: application/json' \ -# -d '{"question": "total order amount per customer region"}' +# -d '{"query": "total order amount per customer region"}' # curl http://localhost:8080/status/ import json @@ -32,7 +32,7 @@ from agents.production_agent import ProductionExecutorAgent -def main(question: str = "total order amount per customer region", n_candidates: int = 3): +def main(query: str = "total order amount per customer region", n_candidates: int = 3): schema_agent = SchemaRetrievalAgent() generator = SQLGeneratorAgent() validator = SQLValidatorAgent() @@ -44,12 +44,12 @@ def main(question: str = "total order amount per customer region", n_candidates: # whichever node created it -- here, this workflow's own -- so resolving # it where it was created is always safe, regardless of which node ends # up running the next stage. - schema = json.loads(schema_agent.get_relevant_schema(question=question).value()) + schema = json.loads(schema_agent.get_relevant_schema(question=query).value()) # Stage 2: fan out candidate SQL queries (LLM calls happen inside). candidates = json.loads( generator.generate_candidates( - question=question, schema=schema, n=n_candidates + question=query, schema=schema, n=n_candidates ).value() ) @@ -70,7 +70,7 @@ def main(question: str = "total order amount per customer region", n_candidates: survivors.append(sql) if not survivors: - return {"question": question, "error": "no candidate passed static validation"} + return {"question": query, "error": "no candidate passed static validation"} # Stage 4: execute survivors on the sampled replica, then vote. sample_results = [ @@ -87,7 +87,7 @@ def main(question: str = "total order amount per customer region", n_candidates: ) return { - "question": question, + "question": query, "candidates": candidates, "costs": costs, "survivors": survivors, diff --git a/pyproject.toml b/pyproject.toml index 40cb43a..41169a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ include = ["ventis*"] ventis = [ "templates/**/*", "controller/proto/*.proto", + "controller/utils/aws_pricing_chart.db", ] @@ -62,4 +63,8 @@ allowed-unresolved-imports = [ [dependency-groups] dev = [ "pytest>=9.1.1", + "canyonos", ] + +[tool.uv.sources] +canyonos = { path = "cli", editable = true } diff --git a/tests/test_canyonos_test.py b/tests/test_canyonos_test.py new file mode 100644 index 0000000..d2d64e5 --- /dev/null +++ b/tests/test_canyonos_test.py @@ -0,0 +1,426 @@ +import hashlib +import json +import subprocess + +import pytest + +from canyonos import test as test_cmd +from canyonos import ui, verify + +CONFIG = """\ +agents: + - name: EchoAgent + entrypoint: echo_agent.py + provider: EC2 + replicas: 2 + + - name: Workflow + type: workflow + workflow_file: echo_workflow.py + api_port: 8080 + provider: EC2 + replicas: 1 +""" + + +@pytest.fixture(autouse=True) +def loud(): + """Every test starts with output enabled; `--json` runs flip it and restore it.""" + ui.set_quiet(False) + yield + ui.set_quiet(False) + + +@pytest.fixture +def project(monkeypatch, tmp_path): + car = tmp_path / ".car" + (car / "config").mkdir(parents=True) + (car / "app").mkdir() + (car / "config" / "global_controller.yaml").write_text(CONFIG) + monkeypatch.chdir(tmp_path) + return tmp_path + + +def report(errors=0, warnings=0, findings=(), ventis=False): + return { + "capabilities": {"ventis": ventis}, + "errors": errors, + "warnings": warnings, + "findings": list(findings), + } + + +def finding(check, level="ERROR"): + return {"check": check, "level": level, "path": "config/x.yaml", "line": 1, "summary": "s"} + + +# ------------------------------------------------------------------ # +# Locating the porting validator # +# ------------------------------------------------------------------ # + + +def test_validator_prefers_the_project_skill(monkeypatch, project, tmp_path): + codex = tmp_path / "codex-skill" + codex.mkdir() + (codex / "validate.py").write_text("") + local = project / ".claude" / "skills" / "porting-to-canyonos" + local.mkdir(parents=True) + (local / "validate.py").write_text("") + + monkeypatch.setattr( + verify, + "AGENTS", + { + "claude": {"skill_dirs": {"local": ".claude/skills/porting-to-canyonos"}}, + "codex": {"skill_dirs": {"global": str(codex)}}, + }, + ) + assert verify._find_validator(str(project)) == str(local / "validate.py") + + +def test_validator_falls_back_to_the_codex_skill(monkeypatch, project, tmp_path): + codex = tmp_path / "codex-skill" + codex.mkdir() + (codex / "validate.py").write_text("") + + monkeypatch.setattr( + verify, + "AGENTS", + { + "claude": {"skill_dirs": {"local": ".claude/skills/porting-to-canyonos"}}, + "codex": {"skill_dirs": {"global": str(codex)}}, + }, + ) + assert verify._find_validator(str(project)) == str(codex / "validate.py") + + +def test_validator_is_fetched_when_nothing_is_installed(monkeypatch, project, tmp_path): + cache = tmp_path / "cache" + monkeypatch.setattr(verify, "AGENTS", {}) + monkeypatch.setattr(verify, "SKILL_CACHE_DIR", str(cache)) + + def fake_install(dest): + assert dest == str(cache) + cache.mkdir() + (cache / "validate.py").write_text("") + return True + + monkeypatch.setattr(verify, "install_skill", fake_install) + assert verify._find_validator(str(project)) == str(cache / "validate.py") + + +def test_a_validator_that_cannot_be_fetched_does_not_stop_the_run(monkeypatch, project, tmp_path): + monkeypatch.setattr(verify, "AGENTS", {}) + monkeypatch.setattr(verify, "SKILL_CACHE_DIR", str(tmp_path / "empty-cache")) + monkeypatch.setattr(verify, "install_skill", lambda _dest: False) + + summary = verify.verify_build_artifact(str(project)) + + assert summary == {"errors": 0, "warnings": 0, "findings": [], "stale": []} + + +# ------------------------------------------------------------------ # +# Reading the validator's report # +# ------------------------------------------------------------------ # + + +def test_validator_errors_fail_the_phase(monkeypatch, project): + monkeypatch.setattr(verify, "_find_validator", lambda _root: "/validate.py") + monkeypatch.setattr( + verify, "_run_validator", lambda *_: report(errors=1, findings=[finding("V002")]) + ) + + with pytest.raises(verify.VerificationError): + verify.verify_build_artifact(str(project)) + + +def test_validator_warnings_pass(monkeypatch, project): + monkeypatch.setattr(verify, "_find_validator", lambda _root: "/validate.py") + monkeypatch.setattr( + verify, + "_run_validator", + lambda *_: report(warnings=1, findings=[finding("V018", "WARN")]), + ) + + summary = verify.verify_build_artifact(str(project)) + + assert (summary["errors"], summary["warnings"]) == (0, 1) + + +def test_rules_needing_ventis_are_dropped_when_it_is_not_importable(monkeypatch, project): + monkeypatch.setattr(verify, "_find_validator", lambda _root: "/validate.py") + monkeypatch.setattr( + verify, + "_run_validator", + lambda *_: report(errors=1, findings=[finding("V030"), finding("V031", "INFO")]), + ) + + summary = verify.verify_build_artifact(str(project)) + + assert summary["errors"] == 0 + assert summary["findings"] == [] + + +def test_rules_needing_ventis_are_kept_when_it_is_importable(monkeypatch, project): + monkeypatch.setattr(verify, "_find_validator", lambda _root: "/validate.py") + monkeypatch.setattr( + verify, + "_run_validator", + lambda *_: report(errors=1, findings=[finding("V030")], ventis=True), + ) + + with pytest.raises(verify.VerificationError): + verify.verify_build_artifact(str(project)) + + +def test_a_missing_car_directory_is_an_error(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + with pytest.raises(verify.VerificationError, match="Run `canyonos build` first"): + verify.verify_build_artifact(str(tmp_path)) + + +# ------------------------------------------------------------------ # +# Source drift # +# ------------------------------------------------------------------ # + + +def write_porting_state(project, entries): + (project / ".car" / "config" / ".porting-state.json").write_text( + json.dumps({"version": 1, "source_files": entries}) + ) + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_unchanged_sources_are_not_reported_as_stale(project): + source = project / "echo_agent.py" + source.write_text("x = 1\n") + write_porting_state(project, {"echo_agent.py": sha256(source)}) + + assert verify._stale_sources(str(project), str(project / ".car")) == [] + + +def test_changed_and_deleted_sources_are_reported(project): + source = project / "echo_agent.py" + source.write_text("x = 2\n") + write_porting_state( + project, {"echo_agent.py": "0" * 64, "gone.py": "0" * 64} + ) + + assert verify._stale_sources(str(project), str(project / ".car")) == [ + "echo_agent.py", + "gone.py", + ] + + +def test_the_skills_own_files_are_not_reported_as_drift(project): + write_porting_state(project, {".claude/skills/porting-to-canyonos/SKILL.md": "0" * 64}) + + assert verify._stale_sources(str(project), str(project / ".car")) == [] + + +def test_a_hand_written_artifact_has_no_state_to_compare(project): + assert verify._stale_sources(str(project), str(project / ".car")) == [] + + +# ------------------------------------------------------------------ # +# Runtime verification # +# ------------------------------------------------------------------ # + + +@pytest.fixture +def runtime(monkeypatch): + monkeypatch.setattr(verify.gc, "workflow_endpoints", lambda _port: []) + + def install(images, containers): + monkeypatch.setattr(verify, "_built_images", lambda: set(images)) + monkeypatch.setattr(verify, "_running_containers", lambda: list(containers)) + + return install + + +ALL_UP = [ + "ventis-local-echoagent-0", + "ventis-local-echoagent-1", + "ventis-local-workflow-0", +] + + +def test_a_complete_deploy_passes(project, runtime): + runtime({"ventis-echoagent", "ventis-workflow"}, ALL_UP) + + result = verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) + + assert [(a["name"], a["running"], a["expected"]) for a in result["agents"]] == [ + ("EchoAgent", 2, 2), + ("Workflow", 1, 1), + ] + + +def test_a_short_replica_count_fails(project, runtime): + runtime({"ventis-echoagent", "ventis-workflow"}, ALL_UP[1:]) + + with pytest.raises(verify.VerificationError, match="1 of 2 replicas"): + verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) + + +def test_an_image_that_was_never_built_fails(project, runtime): + runtime({"ventis-workflow"}, ["ventis-local-workflow-0"]) + + with pytest.raises(verify.VerificationError, match="ventis-echoagent was never built"): + verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) + + +def test_the_workflow_endpoint_falls_back_to_the_configured_port(project, runtime): + runtime({"ventis-echoagent", "ventis-workflow"}, ALL_UP) + + result = verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) + + assert result["agents"][0]["endpoint"] is None + assert result["agents"][1]["endpoint"] == "127.0.0.1:8080" + + +# ------------------------------------------------------------------ # +# The command itself # +# ------------------------------------------------------------------ # + + +@pytest.fixture +def deployable(monkeypatch, project): + """A project where every step past the build check succeeds unless overridden.""" + calls = {"post_deploy": 0, "quit": 0} + + monkeypatch.setattr(test_cmd, "verify_build_artifact", lambda *a: {"warnings": 0, "stale": []}) + monkeypatch.setattr(test_cmd, "run_init", lambda banner=True: None) + monkeypatch.setattr(test_cmd, "run_sync", lambda: True) + monkeypatch.setattr(test_cmd, "load_state", lambda: {"container_id": "abc", "port": 8000}) + monkeypatch.setattr(test_cmd, "_port_in_use", lambda _port: False) + monkeypatch.setattr(test_cmd, "_wait_for_workflow", lambda *a: None) + monkeypatch.setattr(test_cmd, "verify_runtime", lambda *a: {"agents": []}) + monkeypatch.setattr(test_cmd, "workflow_targets", lambda *a: [("Workflow", "127.0.0.1", 8080)]) + monkeypatch.setattr(test_cmd, "_send_query", lambda *a: "req-1") + monkeypatch.setattr(test_cmd, "_await_result", lambda *a: {"status": "done", "result": {"r": 1}}) + monkeypatch.setattr(test_cmd, "_log_tail", lambda _cid: "boom") + + def post_deploy(*_a, **_k): + calls["post_deploy"] += 1 + + def quit_existing(): + calls["quit"] += 1 + + monkeypatch.setattr(test_cmd, "post_deploy", post_deploy) + monkeypatch.setattr(test_cmd, "quit_existing", quit_existing) + return calls + + +def test_a_passing_run_tears_everything_down(deployable): + assert test_cmd.run_test("hi") == 0 + assert deployable["quit"] == 1 + + +def test_the_provider_is_restored_after_the_run(project, deployable): + config = project / ".car" / "config" / "global_controller.yaml" + + test_cmd.run_test("hi") + + assert config.read_text() == CONFIG + + +def test_an_occupied_api_port_fails_before_the_deploy(monkeypatch, deployable, capsys): + monkeypatch.setattr(test_cmd, "_port_in_use", lambda _port: True) + + assert test_cmd.run_test("hi", as_json=True) == 1 + payload = json.loads(capsys.readouterr().out) + + assert deployable["post_deploy"] == 0 + assert "8080 is already in use" in payload["error"] + + +def test_a_failed_deploy_keeps_the_container_and_reads_its_log(monkeypatch, deployable, capsys): + def boom(*_a): + raise test_cmd._TestFailed("the deploy did not come up") + + monkeypatch.setattr(test_cmd, "_wait_for_workflow", boom) + + assert test_cmd.run_test("hi", as_json=True) == 1 + payload = json.loads(capsys.readouterr().out) + + assert deployable["quit"] == 0 + assert payload["log_tail"] == "boom" + + +def test_a_failure_before_the_deploy_leaves_nothing_running(monkeypatch, deployable, capsys): + monkeypatch.setattr(test_cmd, "run_sync", lambda: False) + + assert test_cmd.run_test("hi", as_json=True) == 1 + payload = json.loads(capsys.readouterr().out) + + assert deployable["quit"] == 1 + assert payload["log_tail"] is None + + +def test_json_mode_prints_one_object_and_nothing_else(deployable, capsys): + assert test_cmd.run_test("a prompt", as_json=True) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["ok"] is True + assert payload["query"] == "a prompt" + assert payload["result"] == {"r": 1} + assert payload["error"] is None + assert [(p["name"], p["ok"]) for p in payload["phases"]] == [ + ("verify_build", True), + ("deploy", True), + ("verify_runtime", True), + ("query", True), + ] + + +def test_a_workflow_error_is_reported_as_a_failure(monkeypatch, deployable, capsys): + monkeypatch.setattr( + test_cmd, "_await_result", lambda *a: {"status": "error", "error": "agent blew up"} + ) + + assert test_cmd.run_test("hi", as_json=True) == 1 + + assert json.loads(capsys.readouterr().out)["error"] == "agent blew up" + + +def test_a_flat_layout_project_skips_the_build_check(monkeypatch, tmp_path, deployable, capsys): + legacy = tmp_path / "legacy" / "config" + legacy.mkdir(parents=True) + (legacy / "global_controller.yaml").write_text(CONFIG) + monkeypatch.chdir(tmp_path / "legacy") + + def unexpected(*_a): + raise AssertionError("the artifact validator should not run without a .car/") + + monkeypatch.setattr(test_cmd, "verify_build_artifact", unexpected) + + assert test_cmd.run_test("hi", as_json=True) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["phases"][0] == { + "name": "verify_build", + "ok": True, + "detail": "skipped: no .car/ artifact", + } + + +# ------------------------------------------------------------------ # +# Docker plumbing # +# ------------------------------------------------------------------ # + + +def test_running_containers_are_filtered_to_the_local_provider(monkeypatch): + seen = [] + + def fake_run(argv, **_): + seen.append(argv) + return subprocess.CompletedProcess(argv, 0, "ventis-local-echoagent-0\n", "") + + monkeypatch.setattr(verify.subprocess, "run", fake_run) + + assert verify._running_containers() == ["ventis-local-echoagent-0"] + assert "name=ventis-local-" in seen[0] diff --git a/tests/test_dashboard_stack.py b/tests/test_dashboard_stack.py index 2e62480..7be86ac 100644 --- a/tests/test_dashboard_stack.py +++ b/tests/test_dashboard_stack.py @@ -16,9 +16,7 @@ def project(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path / "home")) monkeypatch.chdir(tmp_path) for key in ( - "DATABASE_URL", "JWT_SECRET", - "CANYONOS_DATABASE_URL", "CANYONOS_JWT_SECRET", "CANYONOS_REDIS_HOST", "CANYONOS_REDIS_PORT", @@ -26,13 +24,9 @@ def project(monkeypatch, tmp_path): "CANYONOS_WEB_IMAGE", ): monkeypatch.delenv(key, raising=False) - config_dir = tmp_path / "config" - config_dir.mkdir() - config = config_dir / "global_controller.yaml" - config.write_text("database:\n url: postgres://user:password@db.example/canyonos\n") monkeypatch.setattr(dashboard_stack.shutil, "which", lambda _: "/usr/bin/docker") monkeypatch.setattr(dashboard_stack, "_port_is_free", lambda _port: True) - return config + return tmp_path def install_docker(monkeypatch, calls, responses=None): @@ -76,86 +70,21 @@ def test_docker_validation_failures_do_not_pull(monkeypatch, project, prepare, m prepare(monkeypatch, responses) install_docker(monkeypatch, calls, responses) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result == dashboard_stack.ServeResult(False, "validate", message) assert all(command[-1] != "pull" for command in calls) -def test_empty_database_url_fails_validation_but_absent_one_does_not(monkeypatch, project): - project.write_text("database:\n url: ''\n") - calls = [] - install_docker(monkeypatch, calls) - - result = dashboard_stack.run_dashboard(str(project)) - - assert result == dashboard_stack.ServeResult( - False, "validate", "database.url must be a non-empty string" - ) - assert all(command[-1] != "pull" for command in calls) - - -def test_missing_database_section_is_not_a_validation_failure(monkeypatch, project): - project.write_text("") - calls = [] - install_docker(monkeypatch, calls) - - stack = dashboard_stack.validate(str(project)) - - assert stack.database_url is None - - -def test_prepare_omits_database_env_when_not_configured(project): - project.write_text("") - stack = dashboard_stack.DashboardStack( - None, dashboard_stack._state_dir(), Path.cwd() - ) - - managed_env, message = dashboard_stack.prepare(stack) - - assert message == "dashboard state prepared" - assert "CANYONOS_DATABASE_URL" not in managed_env - assert "CANYONOS_DATABASE_URL" not in stack.env_path.read_text() - - -def test_config_substitutes_quoted_dotenv_value_without_replacing_source(monkeypatch, project): - source_line = 'DATABASE_URL="postgres://user:password@db.example/canyonos"\n' - Path.cwd().joinpath(".env").write_text(source_line) - project.write_text("database:\n url: ${DATABASE_URL}\n") - calls = [] - install_docker(monkeypatch, calls) - - stack = dashboard_stack.validate(str(project)) - managed_env, _ = dashboard_stack.prepare(stack) - - assert stack.database_url == "postgres://user:password@db.example/canyonos" - assert managed_env["CANYONOS_DATABASE_URL"] == stack.database_url - assert stack.env_path.read_text().startswith(source_line) -def test_missing_database_url_variable_fails_without_pulling(monkeypatch, project): - project.write_text("database:\n url: ${DATABASE_URL}\n") - calls = [] - install_docker(monkeypatch, calls) - result = dashboard_stack.run_dashboard(str(project)) - - assert result == dashboard_stack.ServeResult( - False, - "validate", - "database.url needs ${DATABASE_URL}, which is not set in the project .env", - ) - assert all(command[-1] != "pull" for command in calls) def test_user_jwt_secret_is_untouched_while_canyonos_secret_is_stable(project): source_line = "JWT_SECRET=user-value\n" Path.cwd().joinpath(".env").write_text(source_line) - stack = dashboard_stack.DashboardStack( - "postgres://user:password@db.example/canyonos", - dashboard_stack._state_dir(), - Path.cwd(), - ) + stack = dashboard_stack.DashboardStack(dashboard_stack._state_dir(), Path.cwd()) first_env, _ = dashboard_stack.prepare(stack) second_env, _ = dashboard_stack.prepare(stack) @@ -166,26 +95,6 @@ def test_user_jwt_secret_is_untouched_while_canyonos_secret_is_stable(project): assert first_env["CANYONOS_JWT_SECRET"] == second_env["CANYONOS_JWT_SECRET"] -def test_process_environment_database_url_wins_over_project_dotenv(monkeypatch, project): - Path.cwd().joinpath(".env").write_text("DATABASE_URL=postgres://from-file/canyonos\n") - monkeypatch.setenv("DATABASE_URL", "postgres://from-process/canyonos") - project.write_text("database:\n url: ${DATABASE_URL}\n") - calls = [] - install_docker(monkeypatch, calls) - - stack = dashboard_stack.validate(str(project)) - - assert stack.database_url == "postgres://from-process/canyonos" - - -def test_unreadable_config_does_not_pull(monkeypatch, project): - calls = [] - install_docker(monkeypatch, calls) - - result = dashboard_stack.run_dashboard(str(project.with_name("missing.yaml"))) - - assert result.message == f"config file is not readable: {project.with_name('missing.yaml')}" - assert all(command[-1] != "pull" for command in calls) def test_state_directory_and_port_validation_failures_do_not_pull(monkeypatch, project, tmp_path): @@ -195,7 +104,7 @@ def test_state_directory_and_port_validation_failures_do_not_pull(monkeypatch, p blocked_state_dir.write_text("not a directory") monkeypatch.setattr(dashboard_stack, "_state_dir", lambda: blocked_state_dir) - state_result = dashboard_stack.run_dashboard(str(project)) + state_result = dashboard_stack.run_dashboard() assert state_result.message == "dashboard state directory is not writable" assert all(command[-1] != "pull" for command in calls) @@ -205,7 +114,7 @@ def test_state_directory_and_port_validation_failures_do_not_pull(monkeypatch, p monkeypatch.setattr(dashboard_stack, "_find_web_port", lambda start=8080, max_attempts=50: (_ for _ in ()).throw( dashboard_stack.PhaseFailure("validate", "no free port found for the dashboard after 50 attempts starting at 8080") )) - port_result = dashboard_stack.run_dashboard(str(project)) + port_result = dashboard_stack.run_dashboard() assert port_result.message == "no free port found for the dashboard after 50 attempts starting at 8080" assert all(command[-1] != "pull" for command in calls) @@ -215,15 +124,11 @@ def test_prepare_preserves_unrelated_env_lines_and_mode(project): Path.cwd().joinpath(".env").write_text( "OTHER=one\n# preserved\nJWT_SECRET=kept-secret\nLAST=two\n" ) - stack = dashboard_stack.DashboardStack( - "postgres://user:password@localhost:5432/canyonos", - dashboard_stack._state_dir(), - Path.cwd(), - ) + stack = dashboard_stack.DashboardStack(dashboard_stack._state_dir(), Path.cwd()) managed_env, message = dashboard_stack.prepare(stack) - assert message == "database host localhost is reachable from the stack as host.docker.internal" + assert message == "dashboard state prepared" env_lines = stack.env_path.read_text().splitlines() assert env_lines[:2] == ["OTHER=one", "# preserved"] assert env_lines[2] == "JWT_SECRET=kept-secret" @@ -232,7 +137,6 @@ def test_prepare_preserves_unrelated_env_lines_and_mode(project): "OTHER", "JWT_SECRET", "LAST", - "CANYONOS_DATABASE_URL", "CANYONOS_JWT_SECRET", "CANYONOS_REDIS_HOST", "CANYONOS_REDIS_PORT", @@ -240,35 +144,11 @@ def test_prepare_preserves_unrelated_env_lines_and_mode(project): "CANYONOS_WEB_IMAGE", "CANYONOS_WEB_PORT", } - assert "CANYONOS_DATABASE_URL=postgres://user:password@host.docker.internal:5432/canyonos" in env_lines assert stack.env_path.stat().st_mode & 0o777 == 0o600 assert stack.state_dir.stat().st_mode & 0o777 == 0o700 assert sorted(path.name for path in stack.state_dir.iterdir()) == ["stack.json"] -def test_prepare_reuses_secret_and_rewrites_only_local_hosts(project): - stack = dashboard_stack.DashboardStack( - "postgres://user:password@localhost/canyonos", - dashboard_stack._state_dir(), - Path.cwd(), - ) - first_env, first_message = dashboard_stack.prepare(stack) - second_env, second_message = dashboard_stack.prepare(stack) - - assert first_message.startswith("database host localhost") - assert second_message.startswith("database host localhost") - assert first_env["CANYONOS_JWT_SECRET"] == second_env["CANYONOS_JWT_SECRET"] - assert ( - first_env["CANYONOS_DATABASE_URL"] - == "postgres://user:password@host.docker.internal/canyonos" - ) - - remote_stack = dashboard_stack.DashboardStack( - "postgres://db.example/canyonos", dashboard_stack._state_dir(), Path.cwd() - ) - remote_env, _ = dashboard_stack.prepare(remote_stack) - assert remote_env["CANYONOS_DATABASE_URL"] == "postgres://db.example/canyonos" - def test_redaction_removes_urls_secrets_and_credentials(): database_url = "postgres://user:password@db.example/canyonos" @@ -297,7 +177,7 @@ def response(argv): return completed(argv) install_docker(monkeypatch, calls, response) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result.ok is False assert result.phase == "start" @@ -328,7 +208,7 @@ def response(argv): return completed(argv) install_docker(monkeypatch, calls, response) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result.phase == "pull" assert "pull unauthorized" in result.message @@ -355,7 +235,7 @@ def urlopen(*_args, **_kwargs): monkeypatch.setattr(dashboard_stack.time, "monotonic", lambda: next(clock)) monkeypatch.setattr(dashboard_stack.time, "sleep", lambda _: None) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result.ok is False assert result.phase == "verify" @@ -383,7 +263,7 @@ def urlopen(endpoint, timeout): return Response() monkeypatch.setattr(dashboard_stack.urllib.request, "urlopen", urlopen) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result == dashboard_stack.ServeResult( True, "verify", "dashboard health checks passed", "http://127.0.0.1:8080" @@ -391,7 +271,7 @@ def urlopen(endpoint, timeout): pull_index = next(index for index, command in enumerate(calls) if command[-1] == "pull") up_index = next(index for index, command in enumerate(calls) if "up" in command) assert pull_index < up_index - assert calls[pull_index][4:6] == ["--env-file", str(project.parent.parent / ".env")] + assert calls[pull_index][4:6] == ["--env-file", str(project / ".env")] assert calls[up_index][-5:] == ["up", "-d", "--wait", "--wait-timeout", "180"] assert endpoints == [ ("http://127.0.0.1:8080/healthz", 5), @@ -426,7 +306,7 @@ def response(argv): lambda *_args, **_kwargs: type("Response", (), {"status": 200, "close": lambda self: None})(), ) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result.ok assert result.url == "http://127.0.0.1:8080" diff --git a/tests/test_deploy_progress.py b/tests/test_deploy_progress.py new file mode 100644 index 0000000..d1c8cf6 --- /dev/null +++ b/tests/test_deploy_progress.py @@ -0,0 +1,235 @@ +import pytest + +from canyonos import deploy as deploy_cmd +from canyonos.deploy import PhaseTracker + + +def drive(lines): + """Feed lines to a tracker, returning (spinners, completions, errored).""" + tracker = PhaseTracker() + spinners, done = [], [] + errored = False + for line in lines: + message, completed, is_error = tracker.feed(line) + if is_error: + errored = True + if message: + spinners.append(message) + if completed: + done.append(completed) + return tracker, spinners, done, errored + + +def test_a_full_run_reports_each_phase_once(): + _, spinners, done, errored = drive( + [ + "INFO:ventis:Generating stub: a.yaml -> a_stub.py\n", + "INFO:ventis:Compiling gRPC proto: a.proto\n", + "INFO:ventis:Building 3 Docker image(s) via `docker buildx bake`.\n", + "#5 [4/7] RUN pip install -r requirements.txt\n", + "INFO:ventis:Build complete.\n", + "INFO:ventis:Deploying from config: config.yaml\n", + "INFO:ventis.controller.global_controller:Redis launched on 1 node(s).\n", + "INFO:ventis.controller.global_controller:Waiting for 2 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller Intent (127.0.0.1:50051) is ready.\n", + "INFO:ventis.controller.global_controller:Controller Metrics (127.0.0.1:50052) is ready.\n", + ] + ) + assert not errored + assert done == ["Build complete", "Redis ready"] + assert "Building 3 images..." in spinners + assert spinners[-1] == "Starting agents (2/2 ready)..." + + +def test_phases_are_matched_in_the_order_the_container_emits_them(): + """Redis and stale-container cleanup are logged by GlobalController.__init__, + which runs before `Deploying from config:` -- so the matcher must not assume + the config line comes first. + """ + _, spinners, done, _ = drive( + [ + "INFO:ventis.controller.global_controller:Checking for stale containers from previous runs...\n", + "INFO:ventis.controller.global_controller:Redis launched on 1 node(s).\n", + "INFO:ventis:Deploying from config: config.yaml\n", + ] + ) + assert done == ["Redis ready"] + assert spinners == ["Cleaning up stale containers...", "Starting deploy..."] + + +def test_repeated_build_lines_collapse_to_one_spinner_update(): + _, spinners, _, _ = drive( + [ + "INFO:ventis:Generating stub: a.yaml -> a_stub.py\n", + "INFO:ventis:Generating stub: b.yaml -> b_stub.py\n", + "INFO:ventis:Generating Docker context for 'b'\n", + ] + ) + assert spinners == ["Generating stubs and Docker contexts..."] + + +def test_a_run_with_nothing_to_build_still_reports_the_phase(): + _, _, done, _ = drive( + [ + "INFO:ventis:No Docker images to build.\n", + "INFO:ventis:Build complete.\n", + ] + ) + assert done == ["No images to build", "Build complete"] + + +def test_agent_progress_counts_up_against_the_announced_total(): + tracker, spinners, _, _ = drive( + [ + "INFO:ventis.controller.global_controller:Waiting for 3 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller A (127.0.0.1:1) is ready.\n", + "INFO:ventis.controller.global_controller:Controller B (127.0.0.1:2) is ready.\n", + ] + ) + assert spinners[-1] == "Starting agents (2/3 ready)..." + assert tracker.replicas_total == 3 + + +def test_replicas_of_one_agent_are_counted_separately(): + """`Controller %s is ready.` logs the agent name, which repeats across that + agent's replicas -- the endpoint is what distinguishes them. + """ + tracker, spinners, _, _ = drive( + [ + "INFO:ventis.controller.global_controller:Waiting for 2 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller Echo (127.0.0.1:50051) is ready.\n", + "INFO:ventis.controller.global_controller:Controller Echo (127.0.0.1:50052) is ready.\n", + ] + ) + assert spinners[-1] == "Starting agents (2/2 ready)..." + assert tracker.agents_ready_message() == ("2 agent(s) ready", True) + + +def test_a_re_read_ready_line_does_not_double_count(): + tracker, _, _, _ = drive( + [ + "INFO:ventis.controller.global_controller:Waiting for 2 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller Echo (127.0.0.1:50051) is ready.\n", + "INFO:ventis.controller.global_controller:Controller Echo (127.0.0.1:50051) is ready.\n", + ] + ) + assert tracker.agents_ready_message() == ( + "Workflow up, but only 1/2 agents reported healthy", + False, + ) + + +def test_coming_up_short_of_the_announced_replicas_is_not_reported_as_success(): + """`_wait_for_healthy` gives up after its timeout and the controller starts + anyway, so the up-marker can arrive with agents still unhealthy. + """ + tracker, _, _, _ = drive( + [ + "INFO:ventis.controller.global_controller:Waiting for 3 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller A (127.0.0.1:1) is ready.\n", + ] + ) + message, all_ready = tracker.agents_ready_message() + assert not all_ready + assert message == "Workflow up, but only 1/3 agents reported healthy" + + +def test_a_run_that_never_announced_replicas_still_reports_ready(): + tracker, _, _, _ = drive(["INFO:ventis:Build complete.\n"]) + assert tracker.agents_ready_message() == ("Workflow ready", True) + + +def test_replicas_ready_without_an_announced_total_still_reports_progress(): + _, spinners, _, _ = drive( + ["INFO:ventis.controller.global_controller:Controller A (127.0.0.1:1) is ready.\n"] + ) + assert spinners == ["Starting agents..."] + + +@pytest.mark.parametrize( + "line", + [ + "ERROR:ventis:Config file not found: missing.yaml\n", + "Traceback (most recent call last):\n", + "ERROR: failed to solve: process \"/bin/sh -c pip install\" did not complete successfully\n", + ], +) +def test_fatal_lines_are_flagged(line): + _, _, _, errored = drive([line]) + assert errored + + +@pytest.mark.parametrize( + "line", + [ + "WARNING:ventis.controller.global_controller:otel.destinations not configured -- no OTel metrics collection will happen.\n", + " Warning: no entrypoint mapping for 'agent'\n", + ], +) +def test_benign_warnings_do_not_trip_the_error_path(line): + _, _, _, errored = drive([line]) + assert not errored + + +def test_the_deploy_is_only_declared_dead_after_two_consecutive_checks(monkeypatch): + """One dropped request shouldn't end a deploy that is merely busy.""" + replies = iter([None, {"running": True}, None, None]) + monkeypatch.setattr(deploy_cmd, "deploy_status", lambda _port: next(replies)) + state = {"port": 1} + + misses = 0 + verdicts = [] + for _ in range(4): + dead, misses = deploy_cmd._deploy_is_dead(state, misses) + verdicts.append(dead) + + # a miss, then a recovery that resets the count, then two misses in a row + assert verdicts == [False, False, False, True] + + +def test_a_running_deploy_is_never_declared_dead(monkeypatch): + monkeypatch.setattr(deploy_cmd, "deploy_status", lambda _port: {"running": True}) + dead, misses = deploy_cmd._deploy_is_dead({"port": 1}, 1) + assert not dead and misses == 0 + + +def test_the_clis_own_status_requests_are_not_shown_or_buffered(monkeypatch): + """The container logs every request the CLI makes to it, so its own polling + lands in the stream it is reading. + """ + shown = [] + monkeypatch.setattr(deploy_cmd.ui, "ok", lambda m: shown.append(m)) + monkeypatch.setattr(deploy_cmd.ui, "warn", lambda m: shown.append(m)) + monkeypatch.setattr(deploy_cmd, "_deploy_summary", lambda *a: ("url", [])) + + lines = deploy_cmd._queued_lines( + iter( + [ + '172.17.0.1 - - [04/Sep/2026 21:00:00] "GET /status HTTP/1.1" 200 -\n', + "INFO:ventis:Build complete.\n", + "INFO:ventis.controller.global_controller:Global controller started, polling every 5s...\n", + ] + ) + ) + summary = deploy_cmd._tail_quiet(lines, {"port": 1}, 8080, serve=False) + + assert summary == ("url", []) + assert shown == ["Build complete", "Workflow ready"] + + +def test_a_build_that_dies_silently_does_not_hang(monkeypatch, capsys): + """A failed build leaves `docker logs -f` open with nothing more to say, so + the wait has to end on /status rather than on the stream closing. + """ + monkeypatch.setattr(deploy_cmd, "_STATUS_POLL_SECONDS", 0.01) + monkeypatch.setattr(deploy_cmd, "_REVEAL_GRACE_SECONDS", 0.5) + monkeypatch.setattr(deploy_cmd, "deploy_status", lambda _p: {"running": False}) + + lines = deploy_cmd._queued_lines(iter(["INFO:ventis:Building 2 Docker image(s) via `x`.\n"])) + # The queue never yields None: the stream stays open, as it does in reality. + lines.put = lambda *a, **k: None + + summary = deploy_cmd._tail_quiet(lines, {"port": 1}, 8080, serve=False) + + assert summary is None + assert "Building 2 Docker image(s)" in capsys.readouterr().out diff --git a/tests/test_integration.py b/tests/test_integration.py index a2e3747..85f0c20 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,7 +10,7 @@ def run_integration_test(): base_url = "http://localhost:8080" print(f"Submitting query to {base_url}/main...") - response = requests.post(f"{base_url}/main", json={"ticker": "MSFT"}) + response = requests.post(f"{base_url}/main", json={"query": "MSFT"}) if response.status_code != 202: print(f"Error submitting request: HTTP {response.status_code}") diff --git a/uv.lock b/uv.lock index 9b86805..8278401 100644 --- a/uv.lock +++ b/uv.lock @@ -53,6 +53,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3d/82/6f8fbbea47b773734ba0199643d2d851e5c2f75bc3699fe99db8af344d96/botocore-1.43.58-py3-none-any.whl", hash = "sha256:f516159f0732da8249206163ccea3bd1f82ad2a9d184fe6ed447e1abdba4330e", size = 15426503, upload-time = "2026-07-28T19:34:53.508Z" }, ] +[[package]] +name = "canyonos" +version = "0.1.5" +source = { editable = "cli" } +dependencies = [ + { name = "pyfiglet" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "ruamel-yaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "pyfiglet" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "ruamel-yaml" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -529,6 +548,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -614,6 +645,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.44.0" @@ -854,6 +894,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] +[[package]] +name = "pyfiglet" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/e3/0a86276ad2c383ce08d76110a8eec2fe22e7051c4b8ba3fa163a0b08c428/pyfiglet-1.0.4.tar.gz", hash = "sha256:db9c9940ed1bf3048deff534ed52ff2dafbbc2cd7610b17bb5eca1df6d4278ef", size = 1560615, upload-time = "2025-08-15T18:32:47.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/5c/fe9f95abd5eaedfa69f31e450f7e2768bef121dbdf25bcddee2cd3087a16/pyfiglet-1.0.4-py3-none-any.whl", hash = "sha256:65b57b7a8e1dff8a67dc8e940a117238661d5e14c3e49121032bd404d9b2b39f", size = 1806118, upload-time = "2025-08-15T18:32:45.556Z" }, +] + [[package]] name = "pygments" version = "2.21.0" @@ -984,6 +1033,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "ruamel-yaml" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/3b/ebda527b56beb90cb7652cb1c7e4f91f48649fbcd8d2eb2fb6e77cd3329b/ruamel_yaml-0.19.1.tar.gz", hash = "sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993", size = 142709, upload-time = "2026-01-02T16:50:31.84Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/0c/51f6841f1d84f404f92463fc2b1ba0da357ca1e3db6b7fbda26956c3b82a/ruamel_yaml-0.19.1-py3-none-any.whl", hash = "sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93", size = 118102, upload-time = "2026-01-02T16:50:29.201Z" }, +] + [[package]] name = "s3transfer" version = "0.19.2" @@ -1172,6 +1243,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "canyonos" }, { name = "pytest" }, ] @@ -1193,7 +1265,10 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=9.1.1" }] +dev = [ + { name = "canyonos", editable = "cli" }, + { name = "pytest", specifier = ">=9.1.1" }, +] [[package]] name = "werkzeug" diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index a1cbb4d..be3bb18 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -134,12 +134,16 @@ def provision_instance(spec, replica_index, next_host_port=None): raise RuntimeError( f"EC2 instance {instance_id} does not have a reachable IP address." ) + # Kept alongside `host` (a private IP inside the VPC): callers outside the + # VPC, such as the CLI printing where to send requests, need this one. + public_host = instance.get("PublicIpAddress") if instance else None redis_port = spec.get( "redis_port", _controller.config.get("redis", {}).get("port", 6379) ) record = { "host": host, + "public_host": public_host, "runtime_id": runtime_id, "redis_host": host, "redis_port": redis_port, @@ -169,7 +173,7 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): f"{host}:{CONTAINER_PORT}", timeout=cfg.get("controller_health_timeout", 180), ) - return { + instance = { "agent_name": spec["name"], "provider": "EC2", "instance_type": spec["instance_type"], @@ -182,6 +186,11 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): "redis_port": str(redis_port), "runtime_id": runtime_id, } + if provisioned.get("public_host"): + instance["public_host"] = provisioned["public_host"] + if spec.get("type") == "workflow": + instance["api_port"] = str(spec.get("api_port", 8080)) + return instance except Exception: terminate_instance(provisioned) raise diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index dda4807..88e4ec3 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -153,6 +153,8 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): } if user: instance["user"] = user + if ctrl_type == "workflow": + instance["api_port"] = str(spec.get("api_port", 8080)) logger.info("Runtime ready: %s -> %s", runtime_id, instance["endpoint"]) return instance diff --git a/ventis/controller/instance_manager.py b/ventis/controller/instance_manager.py index e904c5c..4117fd1 100644 --- a/ventis/controller/instance_manager.py +++ b/ventis/controller/instance_manager.py @@ -128,10 +128,11 @@ def _write_instance(self, instance): "redis_port": str(instance["redis_port"]), "runtime_id": instance["runtime_id"], } - if instance.get("user"): - mapping["user"] = instance["user"] - if instance.get("instance_type"): - mapping["instance_type"] = instance["instance_type"] + # public_host: set by providers whose `host` isn't reachable from outside + # the deployment's network. api_port: workflow replicas only. + for field in ("user", "instance_type", "public_host", "api_port"): + if instance.get(field): + mapping[field] = str(instance[field]) self.redis.hset_multiple(key, mapping) node_redis = self.controller.node_redis.get(instance["host"]) or self.redis diff --git a/ventis/server.py b/ventis/server.py index 8df8b0f..31acba7 100644 --- a/ventis/server.py +++ b/ventis/server.py @@ -3,15 +3,22 @@ import subprocess import sys +import yaml from flask import Flask, jsonify, request +from ventis.cli import _artifact_prefix +from ventis.controller.utils.redis_client import RedisClient + app = Flask("ventis-server") # The project files are copied here (into a named volume) by `canyonos sync` / # `canyonos deploy`. Deploy builds and launches against this path. WORKSPACE_DIR = "/workspace" +DEFAULT_API_PORT = 8080 + _gc_process = None +_config_path = None def _gc_running(): @@ -25,13 +32,17 @@ def new_project(): @app.route("/deploy", methods=["POST"]) def deploy(): - global _gc_process + global _gc_process, _config_path if _gc_running(): return jsonify({"error": "already running"}), 409 data = request.get_json(force=True, silent=True) or {} - config_path = data.get("config_path", "config/global_controller.yaml") + # Resolved with ventis' own artifact-layout rule rather than a second copy + # of it, so a `.car` project works when the client sends no config_path. + config_path = data.get("config_path") or os.path.join( + _artifact_prefix(WORKSPACE_DIR), "config", "global_controller.yaml" + ) full_path = os.path.join(WORKSPACE_DIR, config_path) if not os.path.isfile(full_path): @@ -45,6 +56,7 @@ def deploy(): [sys.executable, "-m", "ventis.cli", "deploy", "-c", config_path], cwd=WORKSPACE_DIR, ) + _config_path = full_path return jsonify({"status": "started", "pid": _gc_process.pid}), 200 @@ -66,5 +78,69 @@ def status(): return jsonify({"running": _gc_running()}), 200 +def _primary_redis(config): + """The Redis the controller writes instance records to: the local node's. + + Mirrors GlobalController._launch_redis_containers(), where a localhost node + is reached through VENTIS_REDIS_HOST when the controller is containerized. + """ + redis_cfg = config.get("redis", {}) + host = redis_cfg.get("host", "localhost") + port = redis_cfg.get("port", 6379) + for agent in config.get("agents") or []: + if str(agent.get("provider", "local")).lower() == "local": + port = agent.get("redis_port", port) + break + if host in ("localhost", "127.0.0.1"): + host = os.environ.get("VENTIS_REDIS_HOST", host) + return RedisClient(host=host, port=int(port)) + + +def _workflow_endpoints(config): + """Address of every running workflow replica, as the caller should reach it.""" + ports = { + agent["name"]: agent.get("api_port", DEFAULT_API_PORT) + for agent in config.get("agents") or [] + if agent.get("type") == "workflow" and agent.get("name") + } + if not ports: + return [] + + redis_client = _primary_redis(config) + endpoints = [] + for key in sorted(redis_client.scan_keys("agent_instance:*")): + record = redis_client.hgetall(key) + name = record.get("agent_name") + if name not in ports: + continue + # public_host wins: `host` is the address the controller routes over, + # which for a workflow on another machine is private to that network. + host = record.get("public_host") or record.get("host") + if not host: + continue + endpoints.append( + { + "name": name, + "host": host, + "port": int(record.get("api_port") or ports[name]), + } + ) + return endpoints + + +@app.route("/endpoints", methods=["GET"]) +def endpoints(): + """Where the deployed workflows answer, so the CLI can print real addresses.""" + if _config_path is None or not os.path.isfile(_config_path): + return jsonify({"workflows": []}), 200 + + try: + with open(_config_path) as f: + config = yaml.safe_load(f) or {} + return jsonify({"workflows": _workflow_endpoints(config)}), 200 + except Exception as e: + return jsonify({"workflows": [], "error": str(e)}), 200 + + if __name__ == "__main__": app.run(host="0.0.0.0", port=8000) From 06700a1e09a24ad31241bab57a1f2e22d39a3cb6 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Sat, 5 Sep 2026 00:57:24 -0700 Subject: [PATCH 30/31] docs(cli): add high-level ARCHITECTURE.md and link from README --- cli/ARCHITECTURE.md | 200 ++++++++++++++++++++++++++++++++++++++++++++ cli/README.md | 6 ++ 2 files changed, 206 insertions(+) create mode 100644 cli/ARCHITECTURE.md diff --git a/cli/ARCHITECTURE.md b/cli/ARCHITECTURE.md new file mode 100644 index 0000000..62f869a --- /dev/null +++ b/cli/ARCHITECTURE.md @@ -0,0 +1,200 @@ +# CanyonOS CLI — Architecture + +## The one idea to keep in your head + +**The CLI does almost nothing. The container does everything.** + +`canyonos` is a thin client. It never builds, compiles, or runs your workflow +itself — it manages a **Global Controller (GC) container**, ships your project +into it, and drives it over a small HTTP API. Everything you see in your +terminal is the CLI *narrating* what the container is doing. + +If you remember only one picture, remember this: + +``` + YOU CLI (host) GLOBAL CONTROLLER (container) + │ │ │ + │ canyonos deploy │ │ + ├──────────────────────▶│ pull + run container │ + │ ├───────────────────────────────▶│ + │ │ copy project in (docker cp) │ + │ ├───────────────────────────────▶│ /workspace + │ │ POST /deploy │ + │ ├───────────────────────────────▶│ ventis build + launch + │ │◀── log stream (docker logs) ───┤ │ + │◀── readable progress ─┤ │ ▼ + │ │ spawns Redis + agents + │ │ (sibling containers) +``` + +--- + +## How the pieces connect + +``` +┌───────────────────────────── your machine ─────────────────────────────┐ +│ │ +│ ┌───────────┐ HTTP :8000 ┌──────────────────────────┐ │ +│ │ canyonos │ ───── /deploy /clean ────▶│ Global Controller │ │ +│ │ CLI │ /status /endpoints │ container │ │ +│ │ │ ───── docker cp ─────────▶│ ├─ /workspace (a copy │ │ +│ │ │ ───── docker logs -f ────▶│ │ of your project) │ │ +│ └─────┬─────┘ │ └─ runs `ventis` │ │ +│ │ └───────────┬──────────────┘ │ +│ │ docker compose │ docker.sock │ +│ ▼ ▼ (spawns siblings)│ +│ ┌───────────────────────┐ ┌───────────────────────────┐ │ +│ │ Dashboard stack │◀── traces ───│ Redis + your agent / │ │ +│ │ web · api · postgres │ (OTLP) │ workflow containers │ │ +│ └───────────────────────┘ └───────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +Three things worth internalizing about this diagram: + +1. **The container talks to the host Docker daemon.** The GC mounts the host's + `docker.sock`, so the Redis and agent/workflow containers it launches are + **siblings on your machine**, not nested inside it. (This is why teardown has + to be explicit — see `stop` vs `quit` below.) +2. **Your project is a *copy*, not a live mount.** Files are `docker cp`'d into + a named volume at `/workspace`. Editing files on the host after a deploy does + **not** reach the running build. +3. **The dashboard is separate.** It's its own compose stack that just *renders* + the OTLP traces your workflow emits — it isn't in the deploy critical path. + +State connecting the CLI to its container is a single file: +`~/.canyonos/state.json` (container id + port). Every command that needs the +container reads it. + +--- + +## The three commands that matter + +### `build` — get your code into CanyonOS shape + +``` + you ──▶ canyonos build ──▶ pick agent (Claude/Codex) + scope + └─▶ fetch the porting skill from GitHub + └─▶ launch your coding agent with it + │ + ▼ + generates .car/ ◀── canyonos-formatted project + (originals untouched) +``` + +A **host-side, agent-driven** step. The CLI installs the CanyonOS porting skill +onto your coding agent and hands it a prompt; the agent produces a `.car/` +folder — the canyonos-ready version of your project plus its config. **No +container is involved yet.** + +### `deploy` — the main path + +``` + canyonos deploy + │ + ├─ 1. start fresh → ensure Docker up, tear down any old controller, + │ pull + run the GC container, save state (previous canyonos init) + │ + ├─ 2. ship code → docker cp your project into /workspace + │ + ├─ 3. trigger → POST /deploy (container runs `ventis`: + │ build stubs/images + launch the workflow) + │ + └─ 4. narrate → tail container logs, boil them down to phases, + and when the workflow reports "up": + • auto-start the dashboard (canyonos serve) + • print where everything lives +``` + +Everything after step 3 happens *inside* the container. The CLI's real job in +step 4 is turning a very noisy log stream (a full image-build transcript, etc.) +into a short, readable progression — and, on failure, revealing the part it had +been hiding so you can see the actual cause. + +When it finishes you get a summary panel: the **dashboard URL** and each +**workflow endpoint** (`POST /main`), using the real address the container +placed the workflow at. + +``` + ┌─ Deploy is live ─────────────────────────────┐ + │ Dashboard http://127.0.0.1:8080 │ + │ POST http://127.0.0.1:8000/main │ + │ body {"query": "..."} │ + └──────────────────────────────────────────────┘ +``` + +### `config` — view or edit settings + +``` + canyonos config ──▶ View → pretty tables of agents / otel / general + └─▶ Change → interactive editor (comments & order preserved) +``` + +The important mental model isn't the editor — it's **what a change costs you**: +canyonos config only allows you to change the config file, changing the source code requires a redeploy. + +``` + change type takes effect by... + ─────────────────────── ─────────────────────────────────── + config value only reloads in place (no rebuild) + workflow *code* changes full redeploy (container holds a copy) +``` + +--- + +## Lifecycle: what stays and what goes + +Because the deploy spawns real sibling containers, "make it stop" has two levels of "stop": + +``` + deploy sibling GC project files + stops? containers? container? (volume)? + ─────────────── ─────── ─────────── ───────── ───────────── + canyonos stop ✅ ✅ keep keep + canyonos quit ✅ ✅ remove remove +``` + +- **`stop`** — pause the show, keep the stage set. Redeploy without re-pulling. +- **`quit`** — full teardown. Removes the container *and* the `/workspace` + volume (your copied files). Every `deploy` quietly does this to any previous + controller, so each deploy starts clean. + +And to observe without changing anything: + +- **`logs`** — re-attach to the same live log stream `deploy` shows. Useful + after you Ctrl+C out of a deploy: the deploy keeps running; you just stopped + *watching*. (Ctrl+C on `logs` likewise only detaches.) + +``` + deploy ──▶ (Ctrl+C) ──▶ still running in the container + │ ▲ + └── logs ─────────────────┘ re-attach anytime +``` + +--- + +## The whole loop, one screen + +``` + cd your-project + │ + ▼ + build port your code → .car/ (opens your coding agent) + │ + ▼ + deploy build + launch in the container (dashboard opens itself) + │ + ├─ status where does the workflow answer? + ├─ config tweak settings (live reload); redeploy for code changes + ├─ logs re-attach to the stream + │ + ▼ + stop halt the deploy, keep container + files + or + quit full teardown, remove everything +``` + +That's the entire system: a thin CLI, one container that does the heavy +lifting, a pile of sibling containers it spawns, and a dashboard watching the +whole thing. diff --git a/cli/README.md b/cli/README.md index 76cd4e1..b72f186 100644 --- a/cli/README.md +++ b/cli/README.md @@ -2,6 +2,12 @@ Lightweight CLI for CanyonOS Serves as a thin API layer, connecting to the global controller container. +## Architecture + +For a full walkthrough of the `build`, `deploy`, and `config` flows — plus how +`logs`, `stop`, and `quit` fit into the container lifecycle — see +[ARCHITECTURE.md](ARCHITECTURE.md). + ## Serve `canyonos serve` starts the local CanyonOS dashboard against the Postgres bundled in its compose From f2a8df77258c9a5d1c0ea4198803dca21bb70704 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 8 Sep 2026 14:01:52 -0700 Subject: [PATCH 31/31] reviewed branch and code, made small changes --- cli/ARCHITECTURE.md | 7 +- cli/README.md | 32 ++++----- cli/canyonos/build.py | 2 + cli/canyonos/config.py | 8 +-- cli/canyonos/constants.py | 20 +++++- cli/canyonos/dashboard.compose.yml | 19 ----- cli/canyonos/dashboard_stack.py | 23 ++++--- cli/canyonos/deploy.py | 54 ++++++--------- cli/canyonos/doctor.py | 2 +- cli/canyonos/gc.py | 2 - cli/canyonos/init.py | 11 +-- cli/canyonos/serve.py | 18 +++-- cli/canyonos/status.py | 4 +- cli/canyonos/test.py | 69 ++++++++----------- cli/canyonos/verify.py | 16 ++--- cli/cli.py | 8 ++- cli/utils/help_screen.py | 7 +- cli/utils/tui.py | 4 +- .../helloworld/config/global_controller.yaml | 1 + .../portfolio/config/global_controller.yaml | 1 + .../text2sql/config/global_controller.yaml | 1 + tests/test_canyonos_test.py | 12 ++-- tests/test_dashboard_stack.py | 6 +- ventis/server.py | 21 +++--- 24 files changed, 163 insertions(+), 185 deletions(-) diff --git a/cli/ARCHITECTURE.md b/cli/ARCHITECTURE.md index 62f869a..07619da 100644 --- a/cli/ARCHITECTURE.md +++ b/cli/ARCHITECTURE.md @@ -45,7 +45,7 @@ If you remember only one picture, remember this: │ ▼ ▼ (spawns siblings)│ │ ┌───────────────────────┐ ┌───────────────────────────┐ │ │ │ Dashboard stack │◀── traces ───│ Redis + your agent / │ │ -│ │ web · api · postgres │ (OTLP) │ workflow containers │ │ +│ │ web · api │ (OTLP) │ workflow containers │ │ │ └───────────────────────┘ └───────────────────────────┘ │ │ │ └─────────────────────────────────────────────────────────────────────────┘ @@ -198,3 +198,8 @@ And to observe without changing anything: That's the entire system: a thin CLI, one container that does the heavy lifting, a pile of sibling containers it spawns, and a dashboard watching the whole thing. + + +### Other Notes: +- The ui import is for styling, logging basic commands in the canyonos theme, nothing else. + diff --git a/cli/README.md b/cli/README.md index b72f186..4313946 100644 --- a/cli/README.md +++ b/cli/README.md @@ -1,6 +1,12 @@ -Lightweight CLI for CanyonOS +CLI for CanyonOS -Serves as a thin API layer, connecting to the global controller container. +This CLI does not contain much logic, instead serving as an API to interface with the canyonos container that deploys and runs your entire workflow + + +## Requirements +Need a coding agent(Claude Code, Codex, Cursor) +Need uv or pip +Need docker and docker compose ## Architecture @@ -10,25 +16,11 @@ For a full walkthrough of the `build`, `deploy`, and `config` flows — plus how ## Serve -`canyonos serve` starts the local CanyonOS dashboard against the Postgres bundled in its compose -stack — it reads no project config, so it takes no arguments. It writes only `CANYONOS_`-prefixed -settings into the current directory's `.env`, leaving every other line unchanged. +`canyonos serve` starts the local CanyonOS dashboard — it reads no project config, so it takes no +arguments. It writes only `CANYONOS_`-prefixed settings into the current directory's `.env`, +leaving every other line unchanged. -## Requirements -Need a coding agent(Claude Code, Codex, Cursor) -Need uv or pip -Need docker and docker compose If you have a workflow running, and want to make a config change, canyonos config automatically would reload the project with your config. If you change the workflow files itself though and want the changes to take effect, you need to redeploy from scratch, running canyonos build for good measure -# Use: canyonos -h -### To Republish to PyPi - -```Terminal -cd cli -# Go into, pyproject.toml, and increment version number -rm -rf dist/ # Removes the old distro, causes conflicts - -uv build -uv publish # Needs PyPi Auth Token, ask Saaketh -``` \ No newline at end of file +# Use: canyonos -h \ No newline at end of file diff --git a/cli/canyonos/build.py b/cli/canyonos/build.py index c9b56b4..93c5625 100644 --- a/cli/canyonos/build.py +++ b/cli/canyonos/build.py @@ -1,6 +1,8 @@ """ Logic for `canyonos build`: install the CanyonOS skill on a coding agent, then launch that agent with a prompt to apply it to the current project. + +This file needs to be hardened in particular, will be iterating on it alot with Nick coming up. """ import os diff --git a/cli/canyonos/config.py b/cli/canyonos/config.py index aec5b12..4a0327a 100644 --- a/cli/canyonos/config.py +++ b/cli/canyonos/config.py @@ -1,5 +1,7 @@ """ Logic for `canyonos config`: view or change project/deploy configuration. + +View merely prints out the config, while change opens up a separate temp screen for easy changes. """ import os @@ -141,9 +143,7 @@ def run_view_config(config_path=None): def _is_leaf(value): """A value the user edits directly: any scalar, or a list of only scalars. - Lists of mappings (agents, otel.destinations) are containers to drill into; - lists of plain scalars (requirements, security_group_ids) are edited whole - via comma-separated input. + A non-leaf would be a key that hosts more keys, with only the lowest key's hosting a value """ if isinstance(value, dict): return False @@ -242,7 +242,7 @@ def _confirm_delete(screen, node, key, breadcrumb): def _navigate(screen, node, breadcrumb): - """Drill into a mapping/sequence. Returns True if any value was changed or + """Go into a mapping/sequence. Returns True if any value was changed or deleted, None if the user backed out of this level, or QUIT_ACTION if the user quit (which unwinds the whole session from any depth).""" while True: diff --git a/cli/canyonos/constants.py b/cli/canyonos/constants.py index 455e860..f1aa9d1 100644 --- a/cli/canyonos/constants.py +++ b/cli/canyonos/constants.py @@ -1,4 +1,6 @@ -"""Shared helpers for the canyonos CLI.""" +"""Shared helpers for the canyonos CLI. + +Config/data layer. Holds shared values and parsing helpers""" import os @@ -6,9 +8,11 @@ from ruamel.yaml import YAML DEFAULT_API_PORT = 8080 +DEFAULT_DASHBOARD_PORT = 8081 # The workflow entrypoint is always exposed as POST /main with a {"query": ...} # body, regardless of what the workflow function is called in the project. +# This should be fixed later, keeping it like this for now though WORKFLOW_ROUTE = "main" @@ -32,6 +36,20 @@ def workflow_api_port(config_path): return None +def dashboard_port(config_path): + """Host port the local dashboard prefers to start on, falling back to the default.""" + try: + with open(config_path) as f: + config = yaml.safe_load(f) or {} + except (OSError, yaml.YAMLError): + return DEFAULT_DASHBOARD_PORT + + for agent in config.get("agents") or []: + if agent.get("type") == "workflow": + return agent.get("dashboard_port", DEFAULT_DASHBOARD_PORT) + return DEFAULT_DASHBOARD_PORT + + def workspace_relative(config_path): """`config_path` relative to the cwd, or None if it falls outside it. diff --git a/cli/canyonos/dashboard.compose.yml b/cli/canyonos/dashboard.compose.yml index 0689749..00416f4 100644 --- a/cli/canyonos/dashboard.compose.yml +++ b/cli/canyonos/dashboard.compose.yml @@ -1,24 +1,6 @@ services: - # Fast-path bundled DB, hardcoded creds -- fine for local dev, not for anything real. - db: - image: postgres:16-alpine - environment: - POSTGRES_USER: canyonos - POSTGRES_PASSWORD: canyonos - POSTGRES_DB: canyonos - healthcheck: - test: ["CMD-SHELL", "pg_isready -U canyonos"] - interval: 2s - timeout: 3s - retries: 20 - ports: - - "127.0.0.1:5432:5432" - api: image: ${CANYONOS_API_IMAGE} - depends_on: - db: - condition: service_healthy # Published on all interfaces (not just 127.0.0.1) so a GC container can # actually reach this via host.docker.internal -- Docker's host-gateway # route doesn't reach ports bound to loopback only, so a 127.0.0.1 bind @@ -28,7 +10,6 @@ services: ports: - "3000:3000" environment: - DATABASE_URL: postgresql://canyonos:canyonos@db:5432/canyonos JWT_SECRET: ${CANYONOS_JWT_SECRET} CANYONOS_DISABLE_AUTH: "true" CANYONOS_REDIS_HOST: ${CANYONOS_REDIS_HOST} diff --git a/cli/canyonos/dashboard_stack.py b/cli/canyonos/dashboard_stack.py index e26fbcf..38e38a2 100644 --- a/cli/canyonos/dashboard_stack.py +++ b/cli/canyonos/dashboard_stack.py @@ -19,6 +19,8 @@ from pathlib import Path from typing import Callable +from canyonos.constants import DEFAULT_DASHBOARD_PORT + COMPOSE_PROJECT = "canyonos-dashboard" STACK_VERSION = "v0.1.0-rc.2" API_IMAGE = f"ghcr.io/canyoncodecoreai/canyonos-api:{STACK_VERSION}" @@ -48,7 +50,7 @@ def __init__(self, phase: str, message: str): class DashboardStack: state_dir: Path project_dir: Path - web_port: int = 8080 + web_port: int = DEFAULT_DASHBOARD_PORT @property def env_path(self) -> Path: @@ -111,10 +113,10 @@ def _port_is_free(port: int) -> bool: return True -def _find_web_port(start: int = 8080, max_attempts: int = 50) -> int: +def _find_web_port(start: int = DEFAULT_DASHBOARD_PORT, max_attempts: int = 50) -> int: """First free port at or after `start`, so an unrelated process or container - squatting on 8080 (e.g. a deployed Workflow's own api_port) doesn't - hard-block serve. + squatting on the preferred port (e.g. a deployed Workflow's own api_port) + doesn't hard-block serve. """ for port in range(start, start + max_attempts): if _port_is_free(port): @@ -124,7 +126,8 @@ def _find_web_port(start: int = 8080, max_attempts: int = 50) -> int: ) -def validate() -> DashboardStack: +def validate(preferred_port: int = DEFAULT_DASHBOARD_PORT) -> DashboardStack: + """Checks docker is usable and the state dir is writable, then returns a DashboardStack with the port the dashboard should run on.""" if shutil.which("docker") is None: raise PhaseFailure("validate", "docker is not on PATH") @@ -136,9 +139,8 @@ def validate() -> DashboardStack: except OSError: raise PhaseFailure("validate", "docker daemon or socket is unavailable") - # The dashboard reads no project config -- it always runs against the - # bundled Postgres on this machine -- so the project root is just the cwd, - # the same assumption sync/clean/build already make. + # The dashboard reads no project config, so the project root is just the + # cwd, the same assumption sync/clean/build already make. project_root = Path.cwd() state_dir = _state_dir() @@ -151,7 +153,7 @@ def validate() -> DashboardStack: except OSError: raise PhaseFailure("validate", "dashboard state directory is not writable") - web_port = _existing_dashboard_port() or _find_web_port() + web_port = _existing_dashboard_port() or _find_web_port(preferred_port) return DashboardStack(state_dir, project_root, web_port) @@ -356,6 +358,7 @@ def _cleanup(stack: DashboardStack, manifest: Path) -> None: def run_dashboard( phase_reporter: Callable[[str, str], None] | None = None, + preferred_port: int = DEFAULT_DASHBOARD_PORT, ) -> ServeResult: def report(result: ServeResult) -> None: if phase_reporter is not None: @@ -369,7 +372,7 @@ def report(result: ServeResult) -> None: had_containers = False with ExitStack() as resources: try: - stack = validate() + stack = validate(preferred_port) report(ServeResult(True, "validate", "dashboard prerequisites validated")) managed_env, prepare_message = prepare(stack) diff --git a/cli/canyonos/deploy.py b/cli/canyonos/deploy.py index 048cd3d..6412cfc 100644 --- a/cli/canyonos/deploy.py +++ b/cli/canyonos/deploy.py @@ -38,12 +38,10 @@ from canyonos.serve import serve_dashboard from canyonos.sync import run_sync -LOCAL_HOSTS = ("127.0.0.1", "localhost") - -# Logged exactly once by GlobalController.run(), right after `_wait_for_healthy()` -# returns -- the signal that the workflow finished coming up and entered its -# steady-state polling loop. -_WORKFLOW_UP_MARKER = "Global controller started, polling every" +# Test seams: a failed-build test monkeypatches both down so it doesn't have to +# wait out the real poll/grace windows. +_STATUS_POLL_SECONDS = 2.0 +_REVEAL_GRACE_SECONDS = 30.0 # Substrings that mean the in-container deploy hit something fatal. `WARNING:` is # deliberately absent: the OTel-not-configured notice and stub_generator's @@ -72,25 +70,6 @@ ("Docker container(s) across", "Starting agents...", None), ) -_IMAGE_COUNT = re.compile(r"Building (\d+) Docker image\(s\) via") -_REPLICA_COUNT = re.compile(r"Waiting for (\d+) replica\(s\) to become healthy") -# The name repeats across replicas of one agent, so the endpoint is what makes a -# ready line unique. -_READY = re.compile(r"Controller (\S+ \([^)]+\)) is ready\.") - -# Enough to hold a buildx failure block plus a Python traceback; 40 (what -# `canyonos test` tails) truncates both. -_RECENT_LINES = 200 - -# The container logs every request the CLI makes to it, so its own polling shows -# up in the stream it is reading. -_OWN_REQUEST_MARKER = "GET /status HTTP/1.1" - -_STATUS_POLL_SECONDS = 2.0 - -# Upper bound on how long to keep collecting output after a failure is spotted. -_REVEAL_GRACE_SECONDS = 30.0 - class PhaseTracker: """Turns the container's log lines into the handful of events worth showing. @@ -113,18 +92,19 @@ def feed(self, line): if any(marker in line for marker in _ERROR_MARKERS): return None, None, True - count = _IMAGE_COUNT.search(line) + count = re.search(r"Building (\d+) Docker image\(s\) via", line) if count: self.spinner = f"Building {count.group(1)} images..." return self.spinner, None, False - replicas = _REPLICA_COUNT.search(line) + replicas = re.search(r"Waiting for (\d+) replica\(s\) to become healthy", line) if replicas: self.replicas_total = int(replicas.group(1)) self.spinner = self._agent_progress() return self.spinner, None, False - ready = _READY.search(line) + # Matched on the endpoint, since the name repeats across an agent's replicas. + ready = re.search(r"Controller (\S+ \([^)]+\)) is ready\.", line) if ready: self.replicas_ready.add(ready.group(1)) self.spinner = self._agent_progress() @@ -189,7 +169,7 @@ def workflow_targets(gc_port, api_port): targets = [ ( endpoint.get("name"), - "127.0.0.1" if endpoint["host"] in LOCAL_HOSTS else endpoint["host"], + "127.0.0.1" if endpoint["host"] in ("127.0.0.1", "localhost") else endpoint["host"], endpoint["port"], ) for endpoint in workflow_endpoints(gc_port) @@ -201,6 +181,7 @@ def workflow_targets(gc_port, api_port): def _summary_body(dashboard_url, targets): + """ The contents that go inside the deploy panel""" body = Text() body.append("Dashboard ", "dim") if dashboard_url: @@ -219,7 +200,7 @@ def _summary_body(dashboard_url, targets): body.append('{"query": "your question here"}', WHITE) body.append("\npoll ", "dim") body.append(f"{base}/status/", WHITE) - if host not in LOCAL_HOSTS: + if host not in ("127.0.0.1", "localhost"): body.append(f"\n needs inbound TCP {port} open on {host}", "dim") return body @@ -281,7 +262,8 @@ def _tail_verbose(stream, state, api_port, serve): try: for line in stream: print(line, end="") - if summary is None and _WORKFLOW_UP_MARKER in line: + # Logged exactly once, right after the workflow finishes coming up. + if summary is None and "Global controller started, polling every" in line: summary = _deploy_summary(state, api_port, serve) except KeyboardInterrupt: _interrupted(summary) @@ -295,7 +277,9 @@ def _tail_quiet(lines, state, api_port, serve): dropped rather than allow-listed. `-v` and `canyonos logs` still have it all. """ tracker = PhaseTracker() - recent = deque(maxlen=_RECENT_LINES) + # 200 is enough to hold a buildx failure block plus a Python traceback; + # 40 (what `canyonos test` tails) truncates both. + recent = deque(maxlen=200) reached_up_marker = False # The spinner is exited before the summary panel or the dashboard's own @@ -311,7 +295,8 @@ def _tail_quiet(lines, state, api_port, serve): ui.ok(done) if message: spinner.update(message) - if _WORKFLOW_UP_MARKER in line: + # Logged exactly once, right after the workflow finishes coming up. + if "Global controller started, polling every" in line: summary_line, all_ready = tracker.agents_ready_message() (ui.ok if all_ready else ui.warn)(summary_line) reached_up_marker = True @@ -363,7 +348,8 @@ def _drain(lines, state, deadline=None): if line is None: return misses = 0 - if _OWN_REQUEST_MARKER not in line: + # Otherwise the container logs its own polling into the stream being read. + if "GET /status HTTP/1.1" not in line: yield line diff --git a/cli/canyonos/doctor.py b/cli/canyonos/doctor.py index 15c36af..3339797 100644 --- a/cli/canyonos/doctor.py +++ b/cli/canyonos/doctor.py @@ -60,7 +60,7 @@ def _checks(): def run_doctor(): - """Run every check, print a pass/fail checklist, and return True iff all passed.""" + """Run every check, print a pass/fail checklist, and return True if all passed.""" all_ok = True for label, check, fix in _checks(): try: diff --git a/cli/canyonos/gc.py b/cli/canyonos/gc.py index a8778b3..594c08b 100644 --- a/cli/canyonos/gc.py +++ b/cli/canyonos/gc.py @@ -60,8 +60,6 @@ def post_deploy(port, config_path=None): try: return _request(f"http://127.0.0.1:{port}/deploy", "Deploy", data=body, method="POST") except GCError as e: - if e.code == 409: - raise GCError(f"{e}\n{_DEPLOY_CONFLICT}", code=409) from None raise diff --git a/cli/canyonos/init.py b/cli/canyonos/init.py index 6b07f84..69d4eba 100644 --- a/cli/canyonos/init.py +++ b/cli/canyonos/init.py @@ -105,10 +105,6 @@ def ensure_docker_running(timeout=DOCKER_START_TIMEOUT): def pull_image(image=GC_IMAGE): - # Capture output so the rich status spinner isn't clobbered by docker's own - # layer-progress printing -- but surface it on failure (auth, network, - # rate-limit, missing arch, etc. all otherwise look like the same opaque - # "exit status 1"). result = subprocess.run(["docker", "pull", image], capture_output=True, text=True) if result.returncode != 0: raise RuntimeError( @@ -119,10 +115,7 @@ def pull_image(image=GC_IMAGE): def _port_reachable(port, attempts=10, delay=0.5): """ A successful `docker run` only means Docker accepted the port binding -- - not that traffic actually flows. OrbStack's own port-forwarding proxy for - a given port can get stuck (heavy churn on the same port is enough to - trigger it), which looks fine at the Docker level but resets every real - connection. Confirm the container is actually reachable before trusting it. + not that traffic actually flows. Confirm the container is actually reachable before trusting it. """ url = f"http://127.0.0.1:{port}/status" for _ in range(attempts): @@ -176,12 +169,14 @@ def run_container(image=GC_IMAGE, max_attempts=50): def save_state(container_id, port): + """ Writes GC container info to ~/.canyonos/state.json""" os.makedirs(STATE_DIR, exist_ok=True) with open(STATE_PATH, "w") as f: json.dump({"container_id": container_id, "port": port}, f) def load_state(): + """Reads GC container info from ~/.canyonos/state.json""" with open(STATE_PATH) as f: return json.load(f) diff --git a/cli/canyonos/serve.py b/cli/canyonos/serve.py index c96e336..062eb78 100644 --- a/cli/canyonos/serve.py +++ b/cli/canyonos/serve.py @@ -1,21 +1,29 @@ -"""CLI output for the local dashboard stack.""" +""" +CLI output for the local dashboard stack. + +Automatically gets created when deploy runs unless --serve flag is set to false. +""" from canyonos import ui +from canyonos.constants import dashboard_port, default_config_path from .dashboard_stack import ServeResult, run_dashboard def serve_dashboard() -> ServeResult: - """Bring the dashboard up, reporting progress. Returns the stack's result.""" - # Phases drive the spinner while the stack comes up; the trace itself is - # only printed when something fails and the user needs to see how far it got. + """Bring the dashboard up, reporting progress. Returns the stack's result. + + Phases drive the spinner while the stack comes up; the trace itself is + only printed when something fails and the user needs to see how far it got. + """ trace = [] + preferred_port = dashboard_port(default_config_path()) with ui.status("Starting the dashboard...") as spinner: def report(phase: str, message: str) -> None: trace.append((phase, message)) spinner.update(message) - result = run_dashboard(report) + result = run_dashboard(report, preferred_port) if result.ok: return result diff --git a/cli/canyonos/status.py b/cli/canyonos/status.py index 071297e..e65ffc4 100644 --- a/cli/canyonos/status.py +++ b/cli/canyonos/status.py @@ -22,9 +22,7 @@ def run_status(): ui.ok("Deploy is running.") - # Same resolution `deploy` uses, so both report the address the container - # actually placed the workflow at and fall back to the configured api_port - # rather than a guess. + # Same resolution `deploy` uses targets = workflow_targets(state["port"], workflow_api_port(default_config_path())) for name, target_host, target_port in targets: label = f"Workflow {name}" if name else "Workflow" diff --git a/cli/canyonos/test.py b/cli/canyonos/test.py index 2409965..23f08f7 100644 --- a/cli/canyonos/test.py +++ b/cli/canyonos/test.py @@ -1,14 +1,16 @@ """ Logic for `canyonos test`: check a project end to end on this machine. -Four phases, each ending the run if it fails: the `.car/` artifact `canyonos -build` produced is verified statically, the project is deployed locally (every -agent's `provider` rewritten to `local` for the duration, the original file -restored verbatim afterwards), the running containers are checked against what -the config declared, and one prompt is sent to the workflow's `/main` endpoint. +Four phases, each ending the run if it fails: +- The `.car/` artifact `canyonos build` produced is verified statically +- The project is deployed locally (every agent's `provider` rewritten to `local` for the duration, the original file restored verbatim afterwards) +- The running containers are checked against what the config declared +- One prompt is sent to the workflow's `/main` endpoint. A passing run leaves nothing behind. A failing one leaves the Global Controller container up, with the tail of its log, so there is something left to debug. + +This file will also need lots of iteration based on what is needed, will expect it to change alot """ import json @@ -35,17 +37,11 @@ from canyonos.init import load_state, quit_existing, run_init from canyonos.sync import run_sync from canyonos.theme import GREEN, WHITE -from canyonos.verify import ( - ARTIFACT_DIR, - VerificationError, - verify_build_artifact, - verify_runtime, -) +from canyonos.verify import ARTIFACT_DIR, verify_build_artifact, verify_runtime DEFAULT_QUERY = "hello" -# Generous: the first deploy of a project builds every agent image from scratch. -READY_TIMEOUT = 900 -REQUEST_TIMEOUT = 600 +READY_TIMEOUT = 60 +REQUEST_TIMEOUT = 60 SUBMIT_TIMEOUT = 30 POLL_INTERVAL = 2 LOG_TAIL_LINES = 40 @@ -99,9 +95,9 @@ def _wait_for_workflow(gc_port, api_port): if _workflow_ready("127.0.0.1", api_port): return if not (deploy_status(gc_port) or {}).get("running", False): - raise _TestFailed("The deploy stopped before the workflow came up.") + raise RuntimeError("The deploy stopped before the workflow came up.") time.sleep(POLL_INTERVAL) - raise _TestFailed(f"Timed out after {READY_TIMEOUT}s waiting for the workflow to come up.") + raise RuntimeError(f"Timed out after {READY_TIMEOUT}s waiting for the workflow to come up.") def _send_query(host, port, query): @@ -140,10 +136,6 @@ def _log_tail(container_id): return (result.stdout + result.stderr).strip() or None -class _TestFailed(Exception): - """Ends the run early, carrying a message fit for either output mode.""" - - class _Run: """One `canyonos test` invocation: the phases it got through, and what they found.""" @@ -189,10 +181,7 @@ def _verify_build(run, config_path): run.done("skipped: no .car/ artifact") return - try: - run.validation = verify_build_artifact() - except VerificationError as e: - raise _TestFailed(str(e)) from None + run.validation = verify_build_artifact() stale = len(run.validation["stale"]) run.done(f"{run.validation['warnings']} warning(s), {stale} stale source(s)") @@ -202,13 +191,13 @@ def _deploy_locally(run, config_path, api_port): run_init(banner=False) if not run_sync(): - raise _TestFailed("Could not sync the project into the container.") + raise RuntimeError("Could not sync the project into the container.") # Only the gRPC host port is bumped when a port is taken (the local runtime's # launch retry), so an occupied api_port dies 50 attempts later as "no free # port found". `canyonos serve` also starts looking for its web port at 8080. if _port_in_use(api_port): - raise _TestFailed( + raise RuntimeError( f"Port {api_port} is already in use, and the workflow needs it. Free it " f"(`canyonos quit` stops a previous deploy) or change `api_port` in {config_path}." ) @@ -217,7 +206,7 @@ def _deploy_locally(run, config_path, api_port): try: post_deploy(state["port"], config_path) except GCError as e: - raise _TestFailed(str(e)) from None + raise RuntimeError(str(e)) from None run.deploy_started = True _wait_for_workflow(state["port"], api_port) @@ -227,10 +216,7 @@ def _deploy_locally(run, config_path, api_port): def _verify_runtime(run, config_path, gc_port): run.begin("verify_runtime", 3, "Verify runtime") - try: - run.runtime = verify_runtime(config_path, gc_port) - except VerificationError as e: - raise _TestFailed(str(e)) from None + run.runtime = verify_runtime(config_path, gc_port) run.done(f"{len(run.runtime['agents'])} agent(s) up") @@ -238,7 +224,7 @@ def _query(run, gc_port, api_port): run.begin("query", 4, "Query the workflow") targets = workflow_targets(gc_port, api_port) if not targets: - raise _TestFailed("The deploy reported no workflow endpoint to query.") + raise RuntimeError("The deploy reported no workflow endpoint to query.") _, host, port = targets[0] run.endpoint = f"http://{host}:{port}/{WORKFLOW_ROUTE}" @@ -247,14 +233,14 @@ def _query(run, gc_port, api_port): try: request_id = _send_query(host, port, run.query) except OSError as e: - raise _TestFailed(f"Could not reach the workflow at {run.endpoint}: {e}") from None + raise RuntimeError(f"Could not reach the workflow at {run.endpoint}: {e}") from None data = _await_result(host, port, request_id) status = data.get("status") if status == "error": - raise _TestFailed(data.get("error") or "the workflow returned an error.") + raise RuntimeError(data.get("error") or "the workflow returned an error.") if status != "done": - raise _TestFailed(f"The workflow did not finish within {REQUEST_TIMEOUT}s.") + raise RuntimeError(f"The workflow did not finish within {REQUEST_TIMEOUT}s.") run.result = data.get("result") run.done(f"answered in {run.elapsed()}s") @@ -264,15 +250,15 @@ def _run_test(run): """Walk the four phases, restoring the config whatever happens.""" config_path = workspace_relative(default_config_path()) if config_path is None: - raise _TestFailed("Config must be inside the project directory being synced.") + raise RuntimeError("Config must be inside the project directory being synced.") if not os.path.isfile(config_path): - raise _TestFailed(f"No config at {config_path}. Run `canyonos build` first.") + raise RuntimeError(f"No config at {config_path}. Run `canyonos build` first.") _verify_build(run, config_path) api_port = workflow_api_port(config_path) if api_port is None: - raise _TestFailed(f"No agent with `type: workflow` in {config_path}; nothing to test.") + raise RuntimeError(f"No agent with `type: workflow` in {config_path}; nothing to test.") original_config = _force_local_providers(config_path) try: @@ -361,13 +347,12 @@ def run_test(prompt=None, as_json=False): container_live = False try: _run_test(run) - except _TestFailed as e: - run.error = str(e) except KeyboardInterrupt: run.error = "cancelled by user" except RuntimeError as e: - # Docker unreachable, image pull failed, no free port: all carry a - # readable message, and `--json` needs it inside the payload. + # Every phase raises RuntimeError with a message fit for either output + # mode: docker unreachable, validation failure, port in use, workflow + # timeout, etc. `--json` needs it inside the payload either way. run.error = str(e) if run.error is not None: diff --git a/cli/canyonos/verify.py b/cli/canyonos/verify.py index d047885..95fd36a 100644 --- a/cli/canyonos/verify.py +++ b/cli/canyonos/verify.py @@ -9,6 +9,8 @@ -- every image built, every replica up -- because the controller logs a warning and carries on when an agent never becomes healthy, so a workflow that answers is not on its own proof that the deploy is complete. + +This file will also need lots of iteration based on what is needed, will expect it to change alot """ import hashlib @@ -43,10 +45,6 @@ RUNTIME_PREFIX = "ventis-local-" -class VerificationError(Exception): - """A check that should end the run, carrying a message fit to print.""" - - # ------------------------------------------------------------------ # # Build artifact # # ------------------------------------------------------------------ # @@ -151,14 +149,14 @@ def _stale_sources(project_root, artifact_dir): def verify_build_artifact(project_root="."): - """Check the `.car/` tree. Raises VerificationError if it can't be deployed.""" + """Check the `.car/` tree. Raises RuntimeError if it can't be deployed.""" artifact_dir = os.path.join(project_root, ARTIFACT_DIR) config_path = os.path.join(artifact_dir, CONFIG_REL) if not os.path.isfile(config_path) or not os.path.isdir( os.path.join(artifact_dir, SOURCE_DIR) ): - raise VerificationError( + raise RuntimeError( f"No `{ARTIFACT_DIR}/` artifact here (expected {CONFIG_REL} beside " f"{SOURCE_DIR}/). Run `canyonos build` first." ) @@ -192,7 +190,7 @@ def verify_build_artifact(project_root="."): ui.hint(" -> re-run `canyonos build` to bring the artifact back in step") if summary["errors"]: - raise VerificationError( + raise RuntimeError( f"The build artifact has {summary['errors']} validation error(s); fix them " "or re-run `canyonos build`." ) @@ -238,7 +236,7 @@ def _runtime_table(rows): def verify_runtime(config_path, gc_port): - """Check the running deploy against the config. Raises VerificationError on a gap.""" + """Check the running deploy against the config. Raises RuntimeError on a gap.""" with open(config_path) as f: config = yaml.safe_load(f) or {} @@ -287,5 +285,5 @@ def verify_runtime(config_path, gc_port): ui.panel(_runtime_table(rows)) if problems: - raise VerificationError("The deploy is incomplete -- " + "; ".join(problems)) + raise RuntimeError("The deploy is incomplete -- " + "; ".join(problems)) return {"agents": rows} diff --git a/cli/cli.py b/cli/cli.py index a6bd390..c20e3d8 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -49,8 +49,10 @@ def add(name, run): command.set_defaults(func=run) return command - # Note, not tested much, keeping this in the back burner for now while we flesh out the main path + # New-app can be fleshed out more, keeping it bare for now add("new-app", lambda args: run_new_app()) + + # Deploy has three args: -v, -c, --serve deploy = add("deploy", lambda args: run_deploy(args.config, serve=args.serve, verbose=args.verbose)) deploy.add_argument( "-c", @@ -80,6 +82,8 @@ def add(name, run): add("version", lambda args: ui.say(f"canyonos {importlib.metadata.version('canyonos')}")) add("serve", lambda args: sys.exit(run_serve())) add("status", lambda args: run_status()) + + # Test has 2 args: prompt, --json. test = add("test", lambda args: sys.exit(run_test(args.prompt, as_json=args.json))) test.add_argument( "prompt", @@ -101,8 +105,6 @@ def add(name, run): try: args.func(args) except RuntimeError as e: - # Docker unreachable, image pull failed, no free port -- all already - # carry a readable message, so print it rather than a traceback. ui.fail(e) sys.exit(1) diff --git a/cli/utils/help_screen.py b/cli/utils/help_screen.py index 81ea3ca..717e021 100644 --- a/cli/utils/help_screen.py +++ b/cli/utils/help_screen.py @@ -1,4 +1,7 @@ -"""Custom help screen for the canyonos CLI.""" +"""Custom help screen for the canyonos CLI. + +To run type: canyonos -h +""" from rich.panel import Panel from rich.table import Table @@ -16,8 +19,6 @@ ("config", "Configure project settings"), ) -# The three teardown commands differ only in what they leave behind, so each -# description says so explicitly rather than all three reading as "stop stuff". UTIL_COMMANDS = ( ("clean", "Delete the generated .car folder from this project"), ("doctor", "Check Docker, git and a coding agent are all available"), diff --git a/cli/utils/tui.py b/cli/utils/tui.py index 3154062..c903c8a 100644 --- a/cli/utils/tui.py +++ b/cli/utils/tui.py @@ -1,5 +1,7 @@ """ Minimal arrow-key select menu, no dependency beyond the standard library. + +Used by any command that involves selecting options, no other purpose beyond this. """ import os @@ -16,8 +18,6 @@ DELETE_KEYS = ("d", "D") QUIT_KEYS = ("q", "Q") -# The brand green as a raw truecolor escape: this menu writes ANSI directly -# rather than going through rich, but shares the CLI's one palette. _GREEN = "\x1b[38;2;{};{};{}m".format(*(int(GREEN[i:i + 2], 16) for i in (1, 3, 5))) # Sentinel returned (paired with the hovered value) when the delete key is diff --git a/examples/helloworld/config/global_controller.yaml b/examples/helloworld/config/global_controller.yaml index 0b9c194..d1d82ee 100644 --- a/examples/helloworld/config/global_controller.yaml +++ b/examples/helloworld/config/global_controller.yaml @@ -27,6 +27,7 @@ agents: type: workflow redis_port: 6379 api_port: 8080 # Only needed for workflows, defaults to 8080 if not filled + dashboard_port: 8081 # Local dashboard's preferred port, defaults to 8081 if not filled workflow_file: workflow/example_workflow.py provider: EC2 instance_type: t3.micro diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index dbff7bb..205a672 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -72,6 +72,7 @@ agents: - name: Workflow type: workflow api_port: 8080 # Flask REST API port + dashboard_port: 8081 # Local dashboard's preferred port redis_port: 6379 replicas: 1 workflow_file: workflow/portfolio_workflow.py diff --git a/examples/text2sql/config/global_controller.yaml b/examples/text2sql/config/global_controller.yaml index 89642b2..0df046d 100644 --- a/examples/text2sql/config/global_controller.yaml +++ b/examples/text2sql/config/global_controller.yaml @@ -88,6 +88,7 @@ agents: - name: Workflow type: workflow api_port: 8080 # Flask REST API port + dashboard_port: 8081 # Local dashboard's preferred port redis_port: 6379 replicas: 1 workflow_file: workflow/text2sql_workflow.py diff --git a/tests/test_canyonos_test.py b/tests/test_canyonos_test.py index d2d64e5..13f0b68 100644 --- a/tests/test_canyonos_test.py +++ b/tests/test_canyonos_test.py @@ -130,7 +130,7 @@ def test_validator_errors_fail_the_phase(monkeypatch, project): verify, "_run_validator", lambda *_: report(errors=1, findings=[finding("V002")]) ) - with pytest.raises(verify.VerificationError): + with pytest.raises(RuntimeError): verify.verify_build_artifact(str(project)) @@ -169,13 +169,13 @@ def test_rules_needing_ventis_are_kept_when_it_is_importable(monkeypatch, projec lambda *_: report(errors=1, findings=[finding("V030")], ventis=True), ) - with pytest.raises(verify.VerificationError): + with pytest.raises(RuntimeError): verify.verify_build_artifact(str(project)) def test_a_missing_car_directory_is_an_error(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) - with pytest.raises(verify.VerificationError, match="Run `canyonos build` first"): + with pytest.raises(RuntimeError, match="Run `canyonos build` first"): verify.verify_build_artifact(str(tmp_path)) @@ -262,14 +262,14 @@ def test_a_complete_deploy_passes(project, runtime): def test_a_short_replica_count_fails(project, runtime): runtime({"ventis-echoagent", "ventis-workflow"}, ALL_UP[1:]) - with pytest.raises(verify.VerificationError, match="1 of 2 replicas"): + with pytest.raises(RuntimeError, match="1 of 2 replicas"): verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) def test_an_image_that_was_never_built_fails(project, runtime): runtime({"ventis-workflow"}, ["ventis-local-workflow-0"]) - with pytest.raises(verify.VerificationError, match="ventis-echoagent was never built"): + with pytest.raises(RuntimeError, match="ventis-echoagent was never built"): verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) @@ -340,7 +340,7 @@ def test_an_occupied_api_port_fails_before_the_deploy(monkeypatch, deployable, c def test_a_failed_deploy_keeps_the_container_and_reads_its_log(monkeypatch, deployable, capsys): def boom(*_a): - raise test_cmd._TestFailed("the deploy did not come up") + raise RuntimeError("the deploy did not come up") monkeypatch.setattr(test_cmd, "_wait_for_workflow", boom) diff --git a/tests/test_dashboard_stack.py b/tests/test_dashboard_stack.py index 7be86ac..448c1a1 100644 --- a/tests/test_dashboard_stack.py +++ b/tests/test_dashboard_stack.py @@ -266,7 +266,7 @@ def urlopen(endpoint, timeout): result = dashboard_stack.run_dashboard() assert result == dashboard_stack.ServeResult( - True, "verify", "dashboard health checks passed", "http://127.0.0.1:8080" + True, "verify", "dashboard health checks passed", "http://127.0.0.1:8081" ) pull_index = next(index for index, command in enumerate(calls) if command[-1] == "pull") up_index = next(index for index, command in enumerate(calls) if "up" in command) @@ -274,8 +274,8 @@ def urlopen(endpoint, timeout): assert calls[pull_index][4:6] == ["--env-file", str(project / ".env")] assert calls[up_index][-5:] == ["up", "-d", "--wait", "--wait-timeout", "180"] assert endpoints == [ - ("http://127.0.0.1:8080/healthz", 5), - ("http://127.0.0.1:8080/api/healthz", 5), + ("http://127.0.0.1:8081/healthz", 5), + ("http://127.0.0.1:8081/api/healthz", 5), ] diff --git a/ventis/server.py b/ventis/server.py index 31acba7..e9e9561 100644 --- a/ventis/server.py +++ b/ventis/server.py @@ -43,7 +43,11 @@ def deploy(): config_path = data.get("config_path") or os.path.join( _artifact_prefix(WORKSPACE_DIR), "config", "global_controller.yaml" ) - full_path = os.path.join(WORKSPACE_DIR, config_path) + # realpath, not normpath: /workspace holds a copy of the user's project, which may symlink out. + workspace_root = os.path.realpath(WORKSPACE_DIR) + full_path = os.path.realpath(os.path.join(workspace_root, config_path)) + if not full_path.startswith(workspace_root + os.sep): + return jsonify({"error": "config_path must stay inside the workspace"}), 400 if not os.path.isfile(full_path): return jsonify({"error": f"config file not found: {full_path}"}), 400 @@ -79,11 +83,7 @@ def status(): def _primary_redis(config): - """The Redis the controller writes instance records to: the local node's. - - Mirrors GlobalController._launch_redis_containers(), where a localhost node - is reached through VENTIS_REDIS_HOST when the controller is containerized. - """ + """Client for the node Redis holding instance records, as reached from inside the GC container.""" redis_cfg = config.get("redis", {}) host = redis_cfg.get("host", "localhost") port = redis_cfg.get("port", 6379) @@ -97,7 +97,7 @@ def _primary_redis(config): def _workflow_endpoints(config): - """Address of every running workflow replica, as the caller should reach it.""" + """Address of every workflow replica recorded in Redis, as the caller should reach it.""" ports = { agent["name"]: agent.get("api_port", DEFAULT_API_PORT) for agent in config.get("agents") or [] @@ -138,8 +138,11 @@ def endpoints(): with open(_config_path) as f: config = yaml.safe_load(f) or {} return jsonify({"workflows": _workflow_endpoints(config)}), 200 - except Exception as e: - return jsonify({"workflows": [], "error": str(e)}), 200 + except Exception: + # Don't hand the exception message back to the caller -- it can carry + # local paths or Redis details. Log it here, keep the response generic. + app.logger.exception("Failed to resolve workflow endpoints") + return jsonify({"workflows": [], "error": "failed to resolve endpoints"}), 200 if __name__ == "__main__":