From 4d28f95d8bb56f70fb2507642278975948ad448f Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 17:26:35 -0700 Subject: [PATCH 01/27] includes all the files when ventis build --- ventis/cli.py | 2 ++ ventis/stub_generator.py | 73 ++++++++++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 3aceb18..9ffc149 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -274,6 +274,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), + project_dir=project_dir, ) else: @@ -316,6 +317,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, + project_dir=project_dir, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 8c956ef..a571ccc 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -263,8 +263,36 @@ def _format_source(source): return "\n".join(formatted) + "\n" +# Directories ventis build itself generates inside a project -- never swept. +_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} + + +def _sweep_py_files(project_dir): + """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" + swept = [] + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + for fname in files: + if fname.endswith(".py"): + abs_src = os.path.join(root, fname) + rel_dst = os.path.relpath(abs_src, project_dir) + swept.append((abs_src, rel_dst)) + return swept + + +def _stub_destination(stub_file, project_dir): + """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" + basename = os.path.basename(stub_file) + return os.path.join("agents", basename) if project_dir else basename + + def generate_docker( - yaml_path, agent_file, output_dir=None, grpc_stubs_dir=None, stub_files=None + yaml_path, + agent_file, + output_dir=None, + grpc_stubs_dir=None, + stub_files=None, + project_dir=None, ): """ Generate a minimal Docker build context for an agent. @@ -278,6 +306,7 @@ def generate_docker( output_dir: Optional output directory (default: docker_container//). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -301,8 +330,13 @@ def generate_docker( with open(os.path.join(output_dir, "requirements.txt"), "w") as f: f.write(requirements) + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + # Copy general agent files - files_to_copy = [ + files_to_copy += [ # (source_path, destination_filename) (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -322,13 +356,13 @@ def generate_docker( (os.path.join(script_dir, "llm", "bedrock.py"), "bedrock.py"), ] - # Copy provided agent stubs + # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), os.path.basename(stub_file)) + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) ) - + files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) # Copy gRPC generated stubs if they exist @@ -339,7 +373,9 @@ def generate_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") @@ -377,7 +413,12 @@ def generate_docker( def generate_workflow_docker( - workflow_file, stub_files, output_dir=None, grpc_stubs_dir=None, api_port=8080 + workflow_file, + stub_files, + output_dir=None, + grpc_stubs_dir=None, + api_port=8080, + project_dir=None, ): """ Generate a Docker build context for a workflow. @@ -391,6 +432,7 @@ def generate_workflow_docker( stub_files: List of stub file paths to include. output_dir: Optional output directory (default: docker_container/Workflow/). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -417,7 +459,12 @@ def generate_workflow_docker( # ---- Copy source files into the build context ------------------------ workflow_basename = os.path.basename(workflow_file) - files_to_copy = [ + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + + files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -437,9 +484,11 @@ def generate_workflow_docker( ], ] - # Copy stub files + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) + files_to_copy.append( + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -449,7 +498,9 @@ def generate_workflow_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") From 23e3928d01bc5996fd1cd4e086d45212c283de0e Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 18:23:51 -0700 Subject: [PATCH 02/27] fixed some bugs --- ventis/cli.py | 19 ++++++++++++++ ventis/stub_generator.py | 54 +++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 9ffc149..cb9ee0a 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -220,6 +220,23 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) + # Map each stub's basename to its agent's declared entrypoint, so a stub + # overwrites the exact real file it replaces instead of guessing its path. + stub_entrypoints = {} + for agent_cfg in agents: + entrypoint = agent_cfg.get("entrypoint") + if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: + continue + for yaml_path in yaml_files: + import yaml + + with open(yaml_path) as f: + ydata = yaml.safe_load(f) + if ydata.get("agent", {}).get("name") == agent_cfg["name"]: + base_name = os.path.splitext(os.path.basename(yaml_path))[0] + stub_entrypoints[f"{base_name}.py"] = entrypoint + break + # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -275,6 +292,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) else: @@ -318,6 +336,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a571ccc..a56080d 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -271,7 +271,12 @@ def _sweep_py_files(project_dir): """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" swept = [] for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") + and not (root == project_dir and d in _GENERATED_DIRS) + ] for fname in files: if fname.endswith(".py"): abs_src = os.path.join(root, fname) @@ -280,10 +285,15 @@ def _sweep_py_files(project_dir): return swept -def _stub_destination(stub_file, project_dir): - """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" +def _stub_destination(stub_file, stub_entrypoints): + """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" basename = os.path.basename(stub_file) - return os.path.join("agents", basename) if project_dir else basename + entrypoint = stub_entrypoints.get(basename) + if entrypoint: + normalized = entrypoint.replace("\\", "/") + if not normalized.startswith("/") and ".." not in normalized.split("/"): + return entrypoint + return basename def generate_docker( @@ -293,6 +303,7 @@ def generate_docker( grpc_stubs_dir=None, stub_files=None, project_dir=None, + stub_entrypoints=None, ): """ Generate a minimal Docker build context for an agent. @@ -301,12 +312,13 @@ def generate_docker( source files needed to run the agent with its own local controller. Args: - yaml_path: Path to the YAML agent definition. - agent_file: Path to the original Python agent implementation. - output_dir: Optional output directory (default: docker_container//). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - stub_files: Optional list of agent stub files to copy into the context. - project_dir: Optional project root to sweep for extra .py helper files. + yaml_path: Path to the YAML agent definition. + agent_file: Path to the original Python agent implementation. + output_dir: Optional output directory (default: docker_container//). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -360,7 +372,10 @@ def generate_docker( if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) @@ -419,6 +434,7 @@ def generate_workflow_docker( grpc_stubs_dir=None, api_port=8080, project_dir=None, + stub_entrypoints=None, ): """ Generate a Docker build context for a workflow. @@ -428,11 +444,12 @@ def generate_workflow_docker( with its own local controller. Args: - workflow_file: Path to the workflow Python file. - stub_files: List of stub file paths to include. - output_dir: Optional output directory (default: docker_container/Workflow/). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - project_dir: Optional project root to sweep for extra .py helper files. + workflow_file: Path to the workflow Python file. + stub_files: List of stub file paths to include. + output_dir: Optional output directory (default: docker_container/Workflow/). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -487,7 +504,10 @@ def generate_workflow_docker( # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) # Copy gRPC generated stubs if they exist From 95240ca941167a11c6da6cc791ca4a2fb13c7ebe Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 19:17:55 -0700 Subject: [PATCH 03/27] ventis build: sweep project .py files into Docker build contexts generate_docker()/generate_workflow_docker() now recursively sweep every .py file under the project directory into the build context, preserving directory structure, so helper files that aren't declared as an agent entrypoint still make it into the image. Generated dirs (docker_container/, stubs/, grpc_stubs/) are excluded at the project root only, not at every depth. Stub files are placed at their agent's declared entrypoint path (mapped from global_controller.yaml) instead of a hardcoded guess, so a stub overwrites the exact real file it replaces. Guards against absolute and '..'-containing entrypoints, symlinked sources, and symlinked-destination escapes, with warnings on unsafe or unmapped stubs. Co-Authored-By: Claude Sonnet 5 --- ventis/cli.py | 45 +++++++++++++++--------------------- ventis/stub_generator.py | 49 ++++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index cb9ee0a..4c9badf 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -212,6 +212,23 @@ def cmd_build(args): if not yaml_files: logger.warning("No agent YAML files found in %s", agents_dir) + import yaml + + # Looks up a config entry's YAML and to map stubs to entrypoints. + yaml_by_name = {} + for yaml_path in yaml_files: + with open(yaml_path) as f: + name = yaml.safe_load(f).get("agent", {}).get("name") + if name: + yaml_by_name[name] = yaml_path + + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -220,23 +237,6 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) - # Map each stub's basename to its agent's declared entrypoint, so a stub - # overwrites the exact real file it replaces instead of guessing its path. - stub_entrypoints = {} - for agent_cfg in agents: - entrypoint = agent_cfg.get("entrypoint") - if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: - continue - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_cfg["name"]: - base_name = os.path.splitext(os.path.basename(yaml_path))[0] - stub_entrypoints[f"{base_name}.py"] = entrypoint - break - # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -310,16 +310,7 @@ def cmd_build(args): continue # Find matching YAML by agent name - matching_yaml = None - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_name: - matching_yaml = yaml_path - break - + matching_yaml = yaml_by_name.get(agent_name) if not matching_yaml: logger.warning( "No YAML definition found for agent '%s', skipping Docker", diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a56080d..9ea6c99 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -278,24 +278,43 @@ def _sweep_py_files(project_dir): and not (root == project_dir and d in _GENERATED_DIRS) ] for fname in files: - if fname.endswith(".py"): - abs_src = os.path.join(root, fname) + abs_src = os.path.join(root, fname) + if fname.endswith(".py") and not os.path.islink(abs_src): rel_dst = os.path.relpath(abs_src, project_dir) swept.append((abs_src, rel_dst)) return swept def _stub_destination(stub_file, stub_entrypoints): - """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" + """Where to copy a stub so it overwrites the real file it replaces, falling back to flat if that's unsafe.""" basename = os.path.basename(stub_file) entrypoint = stub_entrypoints.get(basename) if entrypoint: normalized = entrypoint.replace("\\", "/") if not normalized.startswith("/") and ".." not in normalized.split("/"): - return entrypoint + return normalized + print(f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat instead") + elif stub_entrypoints: + print(f" Warning: no entrypoint mapping for stub {basename}, placing flat instead") return basename +def _copy_files(output_dir, files_to_copy): + """Copy each (src, dst) pair into output_dir, refusing to write outside it (e.g. via a symlinked destination parent).""" + real_output_dir = os.path.realpath(output_dir) + for src, dst in files_to_copy: + if not os.path.isfile(src): + print(f" Warning: source file not found, skipping: {src}") + continue + dest_path = os.path.join(output_dir, dst) + real_dest = os.path.realpath(dest_path) + if os.path.commonpath([real_output_dir, real_dest]) != real_output_dir: + print(f" Warning: destination escapes build context, skipping: {dst}") + continue + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) + + def generate_docker( yaml_path, agent_file, @@ -386,13 +405,7 @@ def generate_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # Copy the YAML definition too shutil.copy2( @@ -477,9 +490,7 @@ def generate_workflow_docker( workflow_basename = os.path.basename(workflow_file) # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = [] - if project_dir: - files_to_copy += _sweep_py_files(project_dir) + files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), @@ -500,7 +511,7 @@ def generate_workflow_docker( for name in ("gpu_metrics.py", "session_logging.py") ], ] - + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( @@ -516,13 +527,7 @@ def generate_workflow_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading From 69d5405c24485ff51db960ad7843e496d91354ce Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 13:06:31 -0700 Subject: [PATCH 04/27] Fix missing os import in metrics_agent.py Co-Authored-By: Claude Sonnet 5 --- examples/portfolio/agents/metrics_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 374a869..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,6 +8,7 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. +import os import sys import json From 653bee84d8999a233bf71ce1e75e0ae2d1d44057 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 13:23:55 -0700 Subject: [PATCH 05/27] WIP: OTel exporter testing + portfolio merge-conflict fix (pre-pull checkpoint) Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 174 ++++++++++ OTel_Exporter/__init__.py | 0 OTel_Exporter/convert.py | 96 ++++++ OTel_Exporter/db.py | 185 ++++++++++ OTel_Exporter/otel_exporter.py | 98 ++++++ .../helloworld/workflow/example_workflow.py | 4 +- examples/portfolio/agents/advisor_agent.py | 16 +- examples/portfolio/agents/intent_agent.py | 35 -- examples/portfolio/agents/llm_agent.py | 50 --- examples/portfolio/agents/llm_agent.yaml | 14 - examples/portfolio/agents/metrics_agent.py | 1 + .../portfolio/config/global_controller.yaml | 32 +- examples/portfolio/config/policy.yaml | 1 - pyproject.toml | 6 +- requirements.txt | 4 + uv.lock | 320 ++++++++++++++++++ ventis/controller/global_controller.py | 57 +++- ventis/controller/utils/process_supervisor.py | 59 ++++ ventis/deploy.py | 2 +- ventis/stub_generator.py | 9 +- 20 files changed, 1020 insertions(+), 143 deletions(-) create mode 100644 OTel_Exporter/DESIGN.md create mode 100644 OTel_Exporter/__init__.py create mode 100644 OTel_Exporter/convert.py create mode 100644 OTel_Exporter/db.py create mode 100644 OTel_Exporter/otel_exporter.py delete mode 100644 examples/portfolio/agents/llm_agent.py delete mode 100644 examples/portfolio/agents/llm_agent.yaml create mode 100644 ventis/controller/utils/process_supervisor.py diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md new file mode 100644 index 0000000..73c4da2 --- /dev/null +++ b/OTel_Exporter/DESIGN.md @@ -0,0 +1,174 @@ +# OTLP Exporter for Ventis GlobalController — Design + +Status: **implemented (single-table design)**. `GlobalController` writes futures into a +`waiting` table (SQLite); a GC-supervised, GC-restarted OTel Exporter process reads +finished/unsent rows, converts each to an OTel span, and hands it to a real +`BatchSpanProcessor`/`OTLPSpanExporter`. Batching, serialization, and sending are all +OTel SDK code — the only custom pieces are the row→span conversion and durable +sent-tracking. This doc is a design/rationale reference; the actual files +(`otel_exporter.py`, `db.py`, `convert.py`, `ventis/controller/utils/process_supervisor.py`) +are the source of truth for current behavior. + +## Context +Ventis futures need to reach an external OTLP-compatible tracing backend. Design: a +separate OTLP Exporter process, spawned and supervised by GlobalController, that reads +unsent finished future rows from a local SQLite DB, converts them into OTel spans, and +hands them to the OTel SDK's own batching/export machinery, which ships them to an +external OTLP Receiver (out of scope here — assumed to be a separate, already-addressable +service). + +Decisions (final status): +- **Process model**: a true separate OS process, spawned and supervised by + GlobalController (not an in-process thread) — via `ProcessSupervisor` + (`ventis/controller/utils/process_supervisor.py`, built): `register`/`start_all` to + spawn, `check_and_respawn` (called from GC's existing poll tick, guarded on + `self.running` to avoid a shutdown race) to restart it if it ever dies unexpectedly, + `terminate_all` (called from GC's `stop()`) to shut it down cleanly. Rationale: fault + isolation from GC's core polling/health loop and independent restart, at low added + complexity since SQLite is already the entire hand-off boundary between the two. +- **Config**: implemented via a new `otel:` section in `global_controller.yaml` + (`protocol`/`endpoint`/`headers`), *not* by making `otel_exporter.py` itself + config-aware. `GlobalController` translates that section into the OTel SDK's own + standard env vars (`OTEL_EXPORTER_OTLP_PROTOCOL`/`_ENDPOINT`/`_HEADERS`) and passes + them to the exporter subprocess via `ProcessSupervisor.register(..., env=...)`. The + exporter still just constructs `OTLPSpanExporter()` with no explicit args (endpoint + and headers are resolved by the SDK itself from those env vars, same as always) and + reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly, to pick the gRPC vs HTTP exporter + class — the one piece of protocol selection the plain SDK classes don't do on their + own. Deliberately vendor-neutral: no backend name (Postgres, Langfuse, or otherwise) + appears anywhere in `otel_exporter.py`; the destination is 100% deploy-time config, + set once in `global_controller.yaml` and never touched by app code again. The + originally-planned `database.url` repurposing (below, kept for history) was decided + against — env-var configuration is the SDK's own idiomatic mechanism, so no + exporter-side config plumbing was added, only a GC-side YAML→env-var translation. + Does not (yet) support simultaneous multi-destination export — see "Known gaps". +- **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own + SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing + `_poll_controllers` *alongside* (not instead of) the existing + `send_runtime_information` write. Keeps this pipeline's schema/state fully decoupled + from the dashboard/cost table. +- **Two tables collapsed into one**: an earlier version of this design had a second + `queue` table (`waiting` → promote → `queue` → drain → send). Collapsed once it became + clear `BatchSpanProcessor` already provides its own in-memory queue — the only thing a + second table added was durability across the exporter's own process restarts, which a + `sent` column on `waiting` alone provides just as well, with less code. See `db.py`'s + module docstring. +- **Span construction**: settled — spans are built as `ReadableSpan` objects directly + (bypassing `Tracer`/`TracerProvider` entirely, no `IdGenerator` workaround needed for + either `trace_id` or `span_id`). Confirmed working via `ConsoleSpanExporter` during + development and via real (though unreachable) OTLP export attempts. + +## Implementation summary + +### 1. Config +`global_controller.yaml` gains an optional `otel:` section: +```yaml +otel: + protocol: grpc # or http + endpoint: otlp-pg-receiver.railway.internal:4317 + headers: {} # e.g. Authorization: "Basic " for a backend needing auth +``` +`GlobalController._otel_exporter_env()` translates this into +`OTEL_EXPORTER_OTLP_PROTOCOL`/`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` +and hands them to `ProcessSupervisor.register("otel_exporter", ..., env=...)`, which now +supports an `env` param (merged on top of the parent process's own environment, not a +replacement). Omitting `otel:` entirely falls back to whatever ambient env the exporter +subprocess would otherwise inherit, same as before this change. + +`otel_exporter.py` itself reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly (to pick +which `OTLPSpanExporter` class to import — gRPC or HTTP; the plain SDK classes don't +self-select this the way `opentelemetry-instrument`'s auto-config does). Endpoint and +headers are never read directly — `OTLPSpanExporter()` is still constructed with no +explicit args, letting the SDK resolve those from the same env vars itself, exactly as +before this change. `BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000)` +— the flush delay is explicitly overridden from the SDK default (5000ms) to 1000ms; +`max_export_batch_size` is left at the SDK default (512), which already approximates the +original "500 spans" batching ask without any override needed. + +### 2. `OTel_Exporter/otel_exporter.py` +A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM +stays responsive), calling `_send_pending()` each tick: +- `SELECT * FROM waiting WHERE finished_at IS NOT NULL AND (sent IS NULL OR sent = 0)`. +- Per row, each isolated in its own try/except (one malformed row is logged and skipped, + never blocks the rest of the batch): `convert.waiting_row_to_span(row)` → + `_processor.on_end(span)` → `db.mark_sent(future_id)` immediately — atomic per row, not + batched at the end, so a crash mid-poll can't leave an already-sent row unmarked (which + would cause a duplicate send on the next run). +- `_processor` is constructed once at startup; no `TracerProvider` is used at all, since + spans are hand-built and handed straight to the processor via `on_end()`. +- `_processor.shutdown()` on exit, flushing any pending batch. + +### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) +`future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is +the one that maps to `trace_id`. Both are `uuid4().hex` (32 hex chars / 16 bytes); OTel +`trace_id` is 128-bit (16 bytes, fits directly) and `span_id` is 64-bit (8 bytes, needs +truncation). No hashing — just hex-decode and truncate (deterministic, pure): +```python +trace_id = int(row["session_id"], 16) +span_id = int.from_bytes(bytes.fromhex(row["future_id"])[:8], "big") +parent_span_id = int.from_bytes(bytes.fromhex(row["parent_id"])[:8], "big") if row["parent_id"] else None +``` +Spans are assembled as plain `ReadableSpan(name=..., context=SpanContext(...), parent=SpanContext(...) or None, attributes=..., events=..., status=..., start_time=..., end_time=...)` +— no `Tracer`, no `IdGenerator`. Failed rows get a hand-built `exception` `Event` (using +the SDK's own `EXCEPTION_TYPE`/`EXCEPTION_MESSAGE` constants from `opentelemetry.sdk.trace`, +not hardcoded strings — `record_exception()` can't be used retrospectively since there's +no live exception object, only strings) plus `Status(StatusCode.ERROR, description=...)`. + +**Attribute naming**: `model`/`input_token_count`/`output_token_count` are set under the +real, current OTel GenAI semantic-convention keys — `gen_ai.request.model`/ +`gen_ai.usage.input_tokens`/`gen_ai.usage.output_tokens` — verified against the actual +spec (`open-telemetry/semantic-conventions`), not assumed. `cpu`/`gpu`/ +`execution_time_ms`/`queue_time_ms`/`token_count` keep plain names deliberately: none of +them have an OTel GenAI equivalent (cpu/gpu/queue-time are Ventis infra concepts, and +`token_count`, an input+output sum, isn't part of the spec at all — inventing a +`gen_ai.*`-shaped name for any of these would fabricate a standard rather than follow +one. `cached_tokens`/`cache_hit_ratio` exist on the `waiting` row but aren't exported to +attributes at all yet — a separate, pre-existing gap, not touched here. + +### 4. Process supervisor — `ventis/controller/utils/process_supervisor.py` (built) +`ProcessSupervisor`: `register(name, argv, env=None)` declares a process spec (`env`, +when given, is merged on top of — not a replacement for — the parent's own environment); +`start_all()` spawns everything registered; `check_and_respawn()` restarts anything that +exited, replaying the same argv/env (called from GC's `_poll_controllers`, guarded by +`if self.running:` so a SIGTERM mid-tick can't cause it to resurrect a process +`terminate_all()` just intentionally killed); `terminate_all()` terminates every managed +process (all `.terminate()` calls first, then `.wait()` on each, falling back to +`.kill()`), called from GC's `stop()`. Adding a future second daemon is one more +`register()` call — no new spawn/monitor/terminate code needed. + +### 5. Dependencies (all added) +`opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, +`opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` +config work, since `protocol: http` now needs that package importable). + +## Known gaps (not yet built) +- Spans carry no explicit `resource`/`instrumentation_scope` — would show as + `service.name=unknown_service` at a real backend. +- No simultaneous multi-destination export — `otel:` configures exactly one + destination; sending to two backends at once would mean registering a second, + separately-configured `otel_exporter` subprocess (same script, different env), not + something the exporter or its config format do today. +- `waiting` grows unboundedly: sent rows are never pruned, and futures that never finish + (`finished_at` never arrives) also stay forever, invisible and un-expiring. +- `error_name` is always `NULL` — Ventis's own Redis writer never records a distinct + exception-type field, only a message string. +- No committed test suite — all verification during development was ad hoc scripts, not + `pytest` files under `tests/`. +- Never verified against a live OTLP receiver — only against a refused connection + (confirmed the SDK's real retry/error-handling path is exercised correctly). +- No retry-limit/quarantine for a permanently malformed row — it logs an error every poll + forever rather than being given up on. + +## Verification approach used during development +- Row→span conversion: ad hoc scripts asserting deterministic id derivation, correct + parent/child linkage, correct `ERROR` status + `exception` event on failed rows, and + passing hand-built spans through `ConsoleSpanExporter().export([span])` to confirm the + SDK accepts them without error. +- Pipeline correctness: seeded `waiting` with mixes of finished/still-running/malformed/ + failed rows, ran the real `otel_exporter.py` subprocess, and inspected the resulting + `sent` flags and log output directly — including confirming a second run does not + re-send already-sent rows, and that a malformed row is skipped without blocking others. +- Process supervision: unit-tested `ProcessSupervisor` against a dummy process (spawn, + kill, confirm respawn with a new PID, confirm clean `terminate_all`) and + integration-tested it managing the real `otel_exporter.py` process. +- Scoped to the local provider throughout — no EC2 needed. diff --git a/OTel_Exporter/__init__.py b/OTel_Exporter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/OTel_Exporter/convert.py b/OTel_Exporter/convert.py new file mode 100644 index 0000000..8e062a4 --- /dev/null +++ b/OTel_Exporter/convert.py @@ -0,0 +1,96 @@ +"""Convert a `waiting` table row (see db.py) into an OTel ReadableSpan. + +Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects +directly instead of going through Tracer.start_span() -- there's no live tracer here, +futures already finished (sometimes in another process), so this is a historical-row +conversion, not live tracing. See OTel_Exporter/DESIGN.md for why this deviates from +the SDK's usual advice against constructing ReadableSpan by hand. +""" + +from opentelemetry.sdk.trace import EXCEPTION_MESSAGE, EXCEPTION_TYPE, Event, ReadableSpan +from opentelemetry.trace import SpanContext, SpanKind, TraceFlags +from opentelemetry.trace.status import Status, StatusCode + +_SAMPLED = TraceFlags(TraceFlags.SAMPLED) + + +def to_epoch_nanos(unix_seconds): + """Convert a unix-epoch-seconds float (as stored in waiting) to OTel's ns int.""" + if unix_seconds is None: + return None + return round(float(unix_seconds) * 1e9) + + +def waiting_row_to_span(row): + """Convert one waiting row (dict-like, column names as keys) into a ReadableSpan. + + Rows without finished_at are accepted but produce a span with end_time=None -- + filtering to finished rows is the caller's responsibility, not this function's. + """ + # sqlite3.Row supports row["col"] but not row.get("col") -- normalize once so the + # rest of this function can use .get() freely for optional fields. + row = dict(row) + + trace_id = int(row["session_id"], 16) + span_id = int.from_bytes(bytes.fromhex(row["future_id"])[:8], "big") + parent_id = row.get("parent_id") + parent_span_id = ( + int.from_bytes(bytes.fromhex(parent_id)[:8], "big") if parent_id else None + ) + + context = SpanContext( + trace_id=trace_id, span_id=span_id, is_remote=False, trace_flags=_SAMPLED + ) + parent = ( + SpanContext( + trace_id=trace_id, span_id=parent_span_id, is_remote=False, trace_flags=_SAMPLED + ) + if parent_span_id + else None + ) + + events = [] + status = Status(StatusCode.UNSET) + if row["failed"]: + events.append( + Event( + name="exception", + attributes={ + EXCEPTION_TYPE: row.get("error_name") or "RuntimeError", + EXCEPTION_MESSAGE: row.get("error_message") or "", + }, + timestamp=to_epoch_nanos(row.get("finished_at")), + ) + ) + status = Status(StatusCode.ERROR, description=row.get("error_message")) + + # model/input/output use real OTel GenAI semconv names; cpu/gpu/execution_time_ms/ + # queue_time_ms/token_count have no semconv equivalent (Ventis infra concepts, or -- + # for token_count -- a derived sum the spec doesn't define), so they keep plain names + # rather than being forced into a fake gen_ai.* one. See DESIGN.md. + attributes = { + k: v + for k, v in { + "gen_ai.request.model": row.get("model"), + "cpu": row.get("cpu"), + "gpu": row.get("gpu"), + "execution_time_ms": row.get("execution_time_ms"), + "queue_time_ms": row.get("queue_time_ms"), + "gen_ai.usage.input_tokens": row.get("input_token_count"), + "gen_ai.usage.output_tokens": row.get("output_token_count"), + "token_count": row.get("token_count"), + }.items() + if v is not None + } + + return ReadableSpan( + name=row.get("agent_id") or "unknown_agent", + context=context, + parent=parent, + attributes=attributes, + events=events, + status=status, + kind=SpanKind.INTERNAL, + start_time=to_epoch_nanos(row.get("started_at")), + end_time=to_epoch_nanos(row.get("finished_at")), + ) diff --git a/OTel_Exporter/db.py b/OTel_Exporter/db.py new file mode 100644 index 0000000..4bb301d --- /dev/null +++ b/OTel_Exporter/db.py @@ -0,0 +1,185 @@ +"""SQLite schema and writes for the OTel export pipeline's waiting table. + +`waiting` holds future rows as GlobalController observes them (including still-running +ones). There's no separate queue table -- OTel's own BatchSpanProcessor already queues +and batches spans in memory, so the only thing we need to track durably is which rows +have already been sent, which the `sent` column on this same table provides. (An earlier +version of this pipeline had a second `queue` table for that; collapsed away since it +wasn't doing anything BatchSpanProcessor doesn't already do -- see DESIGN.md.) +""" + +import os +import sqlite3 + +from ventis.controller.utils import pricing + +DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "otel_queue.db") + +# Demo-only multipliers for scaling displayed costs; not real recorded costs. Kept +# deliberately standalone/duplicated from telemetry_logging.py's identical constants +# (rather than importing them) so this module has no dependency on it -- keep these in +# sync by hand if the multipliers there ever change. +_TOKEN_COST_MULTIPLIER = 10000 +_SERVER_COST_MULTIPLIER = 100000 + +# Timestamps are stored as unix epoch seconds (matching the Redis future hash fields +# they're read from), not as SQLite datetime strings. Column set mirrors +# runtime_information 1:1 (see telemetry_logging.py) plus this pipeline's own additions +# (error_name/error_message/sent). +_TABLE_COLUMNS = """ + future_id TEXT PRIMARY KEY, + parent_id TEXT, + session_id TEXT NOT NULL, + project_id TEXT, + agent_id TEXT, + model TEXT, + cpu REAL, + gpu REAL, + started_at TIMESTAMP, + finished_at TIMESTAMP, + execution_time_ms INTEGER, + queue_time_ms INTEGER, + input_token_count INTEGER, + output_token_count INTEGER, + token_count INTEGER, + errors INTEGER, + failed BOOLEAN, + server_cost REAL, + token_cost REAL, + total_cost REAL, + cached_tokens INTEGER, + cache_hit_ratio REAL, + error_name TEXT, + error_message TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + sent BOOLEAN DEFAULT 0 +""" + + +def init_db(db_path=DB_PATH): + """Create the waiting table if it doesn't already exist.""" + conn = sqlite3.connect(db_path) + try: + conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") + conn.commit() + finally: + conn.close() + + +# `sent` is deliberately excluded here so re-upserting a waiting row (e.g. GC +# re-writing it from Redis) never resets it back to unsent. +_COLUMNS = [ + "future_id", "parent_id", "session_id", "project_id", "agent_id", "model", + "cpu", "gpu", "started_at", "finished_at", "execution_time_ms", "queue_time_ms", + "input_token_count", "output_token_count", "token_count", "errors", + "failed", "server_cost", "token_cost", "total_cost", + "cached_tokens", "cache_hit_ratio", "error_name", "error_message", +] + +_WAITING_UPSERT = """ + INSERT INTO waiting ({cols}) VALUES ({placeholders}) + ON CONFLICT(future_id) DO UPDATE SET {updates} +""".format( + cols=", ".join(_COLUMNS), + placeholders=", ".join(f":{c}" for c in _COLUMNS), + updates=", ".join(f"{c}=excluded.{c}" for c in _COLUMNS if c != "future_id"), +) + + +def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH): + """Upsert future rows (as returned by telemetry_logging.pull_runtime_information) + into the waiting table. Unlike runtime_information, rows without finished_at are + kept (not skipped) -- that's what "waiting" means here. `redis_client` is only used + to look up the executing agent's instance type for server-cost pricing, mirroring + send_runtime_information; pass None to skip cost lookups (server_cost stays 0).""" + if not rows: + return + conn = sqlite3.connect(db_path) + try: + for raw in rows: + fid = raw.get("future_id") + session_id = raw.get("request_id") + if not fid or not session_id: + continue + agent_id = raw.get("agent") + started_at = float(raw.get("created_at") or 0) or None + finished_at = float(raw["finished_at"]) if raw.get("finished_at") else None + execution_time_ms = ( + round((finished_at - started_at) * 1000) + if finished_at and started_at + else None + ) + input_token_count = int(float(raw.get("input_token_count") or 0)) + output_token_count = int(float(raw.get("output_token_count") or 0)) + token_count = int(float(raw.get("token_count") or 0)) + cached_tokens = int(float(raw.get("input_cache_tokens") or 0)) + + token_cost = ( + pricing.compute_token_cost( + raw.get("model"), input_token_count, output_token_count + ) + * _TOKEN_COST_MULTIPLIER + ) + # Server cost needs an elapsed duration -- only available once finished. + if finished_at and started_at: + server_cost = ( + pricing.compute_server_cost( + redis_client.get(f"agent:{agent_id}:instance_type") + if redis_client is not None and agent_id + else None, + finished_at - started_at, + ) + * _SERVER_COST_MULTIPLIER + ) + else: + server_cost = 0.0 + + conn.execute( + _WAITING_UPSERT, + { + "future_id": fid, + "parent_id": raw.get("parent") or None, + "session_id": session_id, + "project_id": project_id, + "agent_id": agent_id, + "model": raw.get("model"), + "cpu": float(raw.get("cpu_resource") or 0), + "gpu": float(raw.get("gpu_resource") or 0), + "started_at": started_at, + "finished_at": finished_at, + "execution_time_ms": execution_time_ms, + "queue_time_ms": ( + round(float(raw["queue_time"]) * 1000) + if raw.get("queue_time") + else None + ), + "input_token_count": input_token_count, + "output_token_count": output_token_count, + "token_count": token_count, + "errors": int(raw.get("errors") or 0), + "failed": bool(int(raw.get("failed") or 0)), + "server_cost": server_cost, + "token_cost": token_cost, + "total_cost": server_cost + token_cost, + "cached_tokens": cached_tokens, + "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, + "error_name": raw.get("error_name"), + "error_message": raw.get("error_message"), + }, + ) + conn.commit() + finally: + conn.close() + + +def mark_sent(future_id, db_path=DB_PATH): + """Mark one waiting row sent. Call this immediately after successfully handing its + span to the batch processor -- one row, one commit -- so a crash between two rows' + sends can't leave an already-sent row unmarked (which would cause a duplicate send + on the next run).""" + conn = sqlite3.connect(db_path) + try: + conn.execute("UPDATE waiting SET sent = 1 WHERE future_id = ?", (future_id,)) + conn.commit() + finally: + conn.close() diff --git a/OTel_Exporter/otel_exporter.py b/OTel_Exporter/otel_exporter.py new file mode 100644 index 0000000..b1c7f1a --- /dev/null +++ b/OTel_Exporter/otel_exporter.py @@ -0,0 +1,98 @@ +"""Entrypoint for the OTLP Exporter process. + +Each poll tick: read finished, not-yet-sent rows from `waiting`, convert each to a span, +hand it to a BatchSpanProcessor/OTLPSpanExporter, and mark it sent -- batching, OTLP +serialization, and sending are all the SDK's own code, not ours (see DESIGN.md). Each +row's send-and-mark-sent is atomic and happens immediately after its own successful +send, not batched at the end, so a crash mid-poll can't leave an already-sent row +unmarked (which would cause a duplicate send next run). `OTLPSpanExporter()` takes no +explicit endpoint/headers here -- it falls back to the SDK's own standard +`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` env vars, or localhost:4317, +per the SDK's own default behavior. GlobalController sets those env vars (plus +`OTEL_EXPORTER_OTLP_PROTOCOL`, which this module reads itself below to pick the gRPC vs +HTTP class) from `global_controller.yaml`'s `otel:` section when it spawns this process; +this file has no YAML/app-config awareness of its own, only standard OTel env vars -- +see DESIGN.md. +""" + +import logging +import os +import signal +import sqlite3 +import time + +# Protocol is the one thing the SDK's own exporter classes don't self-select from +# OTEL_EXPORTER_OTLP_PROTOCOL -- endpoint/headers/auth stay fully env-var-driven via +# each class's own defaults; see OTel_Exporter/DESIGN.md. +if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").startswith("http"): + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +else: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +import convert +import db + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +_running = True +_processor = None +POLL_INTERVAL_SECONDS = 5 + + +def _handle_shutdown(signum, frame): + global _running + _running = False + + +def _send_pending(): + """Convert and send each finished, not-yet-sent waiting row.""" + conn = sqlite3.connect(db.DB_PATH) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + "SELECT * FROM waiting WHERE finished_at IS NOT NULL " + "AND (sent IS NULL OR sent = 0)" + ).fetchall() + finally: + conn.close() + if not rows: + return + sent_count = 0 + for row in rows: + try: + span = convert.waiting_row_to_span(row) + _processor.on_end(span) + except Exception as e: + logger.error( + "Skipping waiting row %s -- failed to send: %s", row["future_id"], e + ) + continue + db.mark_sent(row["future_id"]) + sent_count += 1 + logger.info("Sent %d span(s) to the batch processor.", sent_count) + + +def main(): + global _processor + signal.signal(signal.SIGTERM, _handle_shutdown) + signal.signal(signal.SIGINT, _handle_shutdown) + db.init_db() + _processor = BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000) + logger.info("OTel exporter process started.") + last_poll = 0 + while _running: + if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + try: + _send_pending() + except Exception as e: + logger.warning("Poll cycle failed (non-fatal): %s", e) + last_poll = time.time() + time.sleep(1) + _processor.shutdown() + logger.info("OTel exporter process exiting.") + + +if __name__ == "__main__": + main() diff --git a/examples/helloworld/workflow/example_workflow.py b/examples/helloworld/workflow/example_workflow.py index 8bd4600..842fe80 100644 --- a/examples/helloworld/workflow/example_workflow.py +++ b/examples/helloworld/workflow/example_workflow.py @@ -15,11 +15,11 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from example_agent_stub import ExampleAgentStub +from example_agent import ExampleAgent def main(name: str = "World"): - agent = ExampleAgentStub() + agent = ExampleAgent() greeting = agent.hello(name=name) return {"greeting": greeting.value()} diff --git a/examples/portfolio/agents/advisor_agent.py b/examples/portfolio/agents/advisor_agent.py index 2db763f..5cc31ae 100644 --- a/examples/portfolio/agents/advisor_agent.py +++ b/examples/portfolio/agents/advisor_agent.py @@ -11,13 +11,10 @@ # If the LLM is unavailable (returns an empty string), it falls back to a # deterministic templated summary so the pipeline still returns. # -# Resource profile: cheap CPU; the LLM cost sits in LLMAgent, not here. +# Resource profile: cheap CPU, single call per request, on the critical path. -import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -from llm_agent import LLMAgent try: from ventis.llm.bedrock import call_bedrock except ImportError: @@ -27,16 +24,14 @@ class AdvisorAgent(object): def __init__(self): self.tools = [self.summarize] - self.llm = LLMAgent() + self.model_id = os.environ.get( + "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" + ) + self.region = os.environ.get("AWS_REGION", "us-east-1") def summarize(self, holdings: dict, metrics: dict, risk: dict) -> str: """Write a short plain-English briefing on the portfolio.""" prompt = self._build_prompt(holdings, metrics, risk) - text = self.llm.complete( - prompt=prompt, max_tokens=400, temperature=0.2 - ).value() - if not text: - print("AdvisorAgent: LLM returned no output; using templated summary.") try: response = call_bedrock( model_id=self.model_id, @@ -48,7 +43,6 @@ def summarize(self, holdings: dict, metrics: dict, risk: dict) -> str: except Exception as e: print(f"AdvisorAgent: Bedrock call failed ({e}); using templated summary.") return self._fallback_summary(metrics, risk) - return text def _build_prompt(self, holdings: dict, metrics: dict, risk: dict) -> str: lines = ["You are a portfolio analyst. Given the figures below, write a " diff --git a/examples/portfolio/agents/intent_agent.py b/examples/portfolio/agents/intent_agent.py index 972c8af..1eb15d7 100644 --- a/examples/portfolio/agents/intent_agent.py +++ b/examples/portfolio/agents/intent_agent.py @@ -7,17 +7,6 @@ # -> {"holdings": {"AAPL": 0.4, "MSFT": 0.35, "NVDA": 0.25}, # "lookback_days": 180} # -<<<<<<< HEAD -# The actual model call lives in the shared LLMAgent (remote, resolved via -# .value()) — this agent only builds the prompt and parses the result, so no -# Bedrock boilerplate lives here. If the LLM is unavailable or returns -# unparseable output, parse() raises: there is no fallback, the request fails -# loudly rather than guessing at the holdings. Weights are renormalized to 1.0. -# -# Resource profile: cheap CPU; the LLM cost sits in LLMAgent, not here. - -import sys -======= # Calls AWS Bedrock (Converse API) via ventis.llm.bedrock -- same pattern as # AdvisorAgent -- so token/cost telemetry gets recorded onto this execution's # future::metrics hash. Configure with env vars: @@ -31,20 +20,14 @@ # Resource profile: cheap CPU, single call per request, on the critical path # before the fan-out. ->>>>>>> remotes/origin/telemetry-signals import os import re import json -<<<<<<< HEAD -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -from llm_agent import LLMAgent -======= try: from ventis.llm.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock ->>>>>>> remotes/origin/telemetry-signals DEFAULT_LOOKBACK_DAYS = 365 @@ -52,15 +35,6 @@ class IntentAgent(object): def __init__(self): self.tools = [self.parse] -<<<<<<< HEAD - self.llm = LLMAgent() - - def parse(self, query: str) -> dict: - """Parse a natural-language portfolio request into holdings + lookback.""" - text = self.llm.complete( - prompt=self._build_prompt(query), max_tokens=300, temperature=0.0 - ).value() -======= self.model_id = os.environ.get( "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" ) @@ -75,7 +49,6 @@ def parse(self, query: str) -> dict: region=self.region, ) text = response["output"]["message"]["content"][0]["text"] ->>>>>>> remotes/origin/telemetry-signals if not text: raise ValueError("IntentAgent: LLM returned no output for the request.") @@ -148,15 +121,7 @@ def _sanitize(self, parsed: dict) -> dict: if __name__ == "__main__": -<<<<<<< HEAD - # Assumes the LLMAgent stub (Future-returning) is on the path, as it is - # inside the deployed pipeline. - agent = IntentAgent() - print(agent.parse( - "Analyze 40% Apple, 35% Microsoft and 25% Nvidia over the last 6 months" -======= agent = IntentAgent() print(agent.parse( query="Analyze 40% Apple, 35% Microsoft and 25% Nvidia over the last 6 months" ->>>>>>> remotes/origin/telemetry-signals )) diff --git a/examples/portfolio/agents/llm_agent.py b/examples/portfolio/agents/llm_agent.py deleted file mode 100644 index d42e4fb..0000000 --- a/examples/portfolio/agents/llm_agent.py +++ /dev/null @@ -1,50 +0,0 @@ -# LLM Agent -# -# Shared inference node. Owns all the AWS Bedrock (Converse API) plumbing so no -# other agent has to carry boto3 boilerplate — they just call complete(prompt) -# and get text back. Configure with env vars: -# BEDROCK_MODEL_ID (default: meta.llama3-8b-instruct-v1:0) -# AWS_REGION (default: us-east-1) -# -# On any failure (no boto3, no creds, model not enabled) it returns an empty -# string; callers decide how to degrade (templated summary, regex parse, etc.). -# -# Resource profile: LLM-bound. This is the only node that talks to Bedrock, so -# it's the natural place to scale inference capacity independently. - -import os - - -class LLMAgent(object): - def __init__(self): - self.tools = [self.complete] - self.model_id = os.environ.get( - "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" - ) - self.region = os.environ.get("AWS_REGION", "us-east-1") - - def complete( - self, prompt: str, max_tokens: int = 400, temperature: float = 0.2 - ) -> str: - """Run a single-turn completion on Bedrock; '' on any failure.""" - try: - import boto3 - - client = boto3.client("bedrock-runtime", region_name=self.region) - response = client.converse( - modelId=self.model_id, - messages=[{"role": "user", "content": [{"text": prompt}]}], - inferenceConfig={ - "maxTokens": max_tokens, - "temperature": temperature, - }, - ) - return response["output"]["message"]["content"][0]["text"] - except Exception as e: - print(f"LLMAgent: Bedrock call failed ({e}).") - return "" - - -if __name__ == "__main__": - agent = LLMAgent() - print(agent.complete("Say hello in one short sentence.", max_tokens=50)) \ No newline at end of file diff --git a/examples/portfolio/agents/llm_agent.yaml b/examples/portfolio/agents/llm_agent.yaml deleted file mode 100644 index ccc0095..0000000 --- a/examples/portfolio/agents/llm_agent.yaml +++ /dev/null @@ -1,14 +0,0 @@ -agent: - name: LLMAgent - functions: - - name: complete - description: Run a single-turn completion on Bedrock; '' on any failure. - arguments: - - name: prompt - type: str - - name: max_tokens - type: int - - name: temperature - type: float - returns: - type: str diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 374a869..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,6 +8,7 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. +import os import sys import json diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index 29e8de9..9a0a9a5 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -6,26 +6,6 @@ # reflect each stage's real cost so the scheduler has placement decisions to make. agents: -<<<<<<< HEAD - # Shared inference node. Owns all Bedrock plumbing; IntentAgent and - # AdvisorAgent delegate their model calls here. LLM-bound — scale replicas - # to match inference demand. - - name: LLMAgent - host: localhost - port: 8075 - redis_port: 6379 - replicas: 1 - resources: - cpu: 1 - memory: 512 - entrypoint: agents/llm_agent.py - - # Stage 0: parse the free-text request into structured holdings + lookback - # window (calls LLMAgent). Cheap CPU, one call per request, on the critical - # path before the fan-out. - - name: IntentAgent - host: localhost - port: 8076 # Stage 0: parse the free-text request into structured holdings + lookback # window (calls Bedrock directly via ventis.llm.bedrock). Cheap CPU, one # call per request, on the critical path before the fan-out. @@ -36,8 +16,6 @@ agents: cpu: 1 memory: 256 entrypoint: agents/intent_agent.py - - # Stage 0: price history fetch. Network/IO-bound, cheap CPU. Called by provider: EC2 instance_type: t3.micro @@ -107,3 +85,13 @@ redis: host: localhost port: 6379 db: 0 + +# EC2 defaults for `provider: EC2` replicas. +ec2: + region: us-east-1 + ami_id: ami-031ff6df47f26b546 + subnet_id: subnet-0638ac6d79d488124 + security_group_ids: + - sg-025daf3a98e06cef3 + ssh_user: ubuntu + ssh_private_key_path: ~/.ssh/ventis_ec2 diff --git a/examples/portfolio/config/policy.yaml b/examples/portfolio/config/policy.yaml index 573c91b..834c88e 100644 --- a/examples/portfolio/config/policy.yaml +++ b/examples/portfolio/config/policy.yaml @@ -14,7 +14,6 @@ rules: - match: {} access: - Workflow - - LLMAgent - IntentAgent - PriceAgent - MetricsAgent diff --git a/pyproject.toml b/pyproject.toml index 2efc1ab..8410b24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,10 @@ dependencies = [ "pyyaml", "flask", "psutil", + "opentelemetry-api>=1.44.0", + "opentelemetry-sdk>=1.44.0", + "opentelemetry-exporter-otlp-proto-grpc>=1.44.0", + "opentelemetry-exporter-otlp-proto-http>=1.44.0", ] [project.scripts] @@ -23,7 +27,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["ventis*"] +include = ["ventis*", "OTel_Exporter*"] [tool.setuptools.package-data] ventis = [ diff --git a/requirements.txt b/requirements.txt index b0dd97e..dd7a254 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,7 @@ ipython sqlalchemy psycopg[binary] psutil +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp-proto-grpc +opentelemetry-exporter-otlp-proto-http diff --git a/uv.lock b/uv.lock index 9b7b188..ba10707 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,11 @@ version = 1 revision = 3 requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version < '3.13'", +] [[package]] name = "async-timeout" @@ -48,6 +53,178 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3d/82/6f8fbbea47b773734ba0199643d2d851e5c2f75bc3699fe99db8af344d96/botocore-1.43.58-py3-none-any.whl", hash = "sha256:f516159f0732da8249206163ccea3bd1f82ad2a9d184fe6ed447e1abdba4330e", size = 15426503, upload-time = "2026-07-28T19:34:53.508Z" }, ] +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/aa/554e2614f38fc34c58ff1d0911ae8535ad2516440d5482d76fe59f1088b0/charset_normalizer-3.5.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa", size = 369072, upload-time = "2026-08-15T08:16:22.964Z" }, + { url = "https://files.pythonhosted.org/packages/03/6d/439231dfc3ccfa6f8c06477b7da2219cbd41a2de3d49084df8ec7b5100f2/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda", size = 251142, upload-time = "2026-08-15T08:16:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/55/53/7d819bd23a00ef45039146fa2cce1daa2f0771e758c5653ee1f6edac91ed/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45", size = 240714, upload-time = "2026-08-15T08:16:26.392Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2c/45847198c16f4b38090cc7423b2b6a9008e438704d8ab413211832498d31/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab", size = 279637, upload-time = "2026-08-15T08:16:27.961Z" }, + { url = "https://files.pythonhosted.org/packages/69/2b/d8be3523ddf9f0b0f3e56d1359034aa10653a4d11564c697f802b4775766/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a", size = 276543, upload-time = "2026-08-15T08:16:29.399Z" }, + { url = "https://files.pythonhosted.org/packages/32/cd/4f564b8f132de25db594efc706897069f016790cea63a5669c9df2675f64/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3", size = 261644, upload-time = "2026-08-15T08:16:30.722Z" }, + { url = "https://files.pythonhosted.org/packages/f5/e3/38b975422534a608f98c360e79c2f07c763d66dd4272300d45fb1fee54b0/charset_normalizer-3.5.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb", size = 259609, upload-time = "2026-08-15T08:16:32.248Z" }, + { url = "https://files.pythonhosted.org/packages/87/bd/fbc24d825c66f1c74f6ccdea3742c3d8354a4888e86d1315a197fee69061/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f", size = 252457, upload-time = "2026-08-15T08:16:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2d/918d0e98a0e679469ed05bb2d90c2088b4d315bb612969d8499f76fb5210/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea", size = 242240, upload-time = "2026-08-15T08:16:35.396Z" }, + { url = "https://files.pythonhosted.org/packages/20/c8/c36f6e0b2dfec351bd38cbc05362697e58bcd073d7dbd95154290c9714ce/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f", size = 280308, upload-time = "2026-08-15T08:16:36.825Z" }, + { url = "https://files.pythonhosted.org/packages/ca/7b/311b3e02e8c4092400c449c850a760d8c45d900983c83a70cc07208c551d/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182", size = 258679, upload-time = "2026-08-15T08:16:38.22Z" }, + { url = "https://files.pythonhosted.org/packages/b9/90/082cc45599c392f28c036a497f49e0634041a785fc3849c80ccf396d096f/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa", size = 277221, upload-time = "2026-08-15T08:16:39.62Z" }, + { url = "https://files.pythonhosted.org/packages/58/ad/b9aecf38d805cbcf84fa94f14c5d972a16561e20296a11dc799a5dcf3763/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818", size = 263799, upload-time = "2026-08-15T08:16:40.885Z" }, + { url = "https://files.pythonhosted.org/packages/b7/23/b38a20598d5a825f85d9d7636860e56ff0db1479f86497a6e485aa9326f7/charset_normalizer-3.5.1-cp310-cp310-win32.whl", hash = "sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20", size = 182037, upload-time = "2026-08-15T08:16:42.198Z" }, + { url = "https://files.pythonhosted.org/packages/d2/21/83fffb77864408b8bf0fe1ca603926401d6f8775a8e150b39aacc9958f8a/charset_normalizer-3.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d", size = 206030, upload-time = "2026-08-15T08:16:43.787Z" }, + { url = "https://files.pythonhosted.org/packages/86/2e/b93135b5034b1157fb29554b0d06d4844ce62282f0e0a14036f93d7ee2e7/charset_normalizer-3.5.1-cp310-cp310-win_arm64.whl", hash = "sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5", size = 185092, upload-time = "2026-08-15T08:16:45.177Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b6/034f6802e9c3f6418966cfabb7db8c9252cc2429c5098f41cc43af804149/charset_normalizer-3.5.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30", size = 363585, upload-time = "2026-08-15T08:16:46.646Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fa/6a7e2a7c4b5451912b8c417732df79574354443592a88d616de03da66ae5/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488", size = 251189, upload-time = "2026-08-15T08:16:48.287Z" }, + { url = "https://files.pythonhosted.org/packages/a4/c8/ab42b07cfd82e919f427fcfaa7c41abae8242833ad1aad66d42bae40b669/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22", size = 239724, upload-time = "2026-08-15T08:16:49.67Z" }, + { url = "https://files.pythonhosted.org/packages/e7/80/b9348b5d3041209f98b4cdad7655766369233f1d533f4f4f7558e9717bec/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731", size = 280078, upload-time = "2026-08-15T08:16:51.228Z" }, + { url = "https://files.pythonhosted.org/packages/82/38/083a24028304bc85bb9e376fed801178423dcbb67495f73b6ea0624e1894/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c", size = 276650, upload-time = "2026-08-15T08:16:52.625Z" }, + { url = "https://files.pythonhosted.org/packages/0d/35/731ac04aa0a097fc1c97f0994c375bdb230c6c96619db794208fe664e9ce/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8", size = 262325, upload-time = "2026-08-15T08:16:54.085Z" }, + { url = "https://files.pythonhosted.org/packages/f5/28/c2028e7021fb89c6e56868ed0e387b8e9aa811abdd2ab3208d6578d2c930/charset_normalizer-3.5.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486", size = 261140, upload-time = "2026-08-15T08:16:55.604Z" }, + { url = "https://files.pythonhosted.org/packages/28/f0/0c0ceec6d98b7daa62e361e418135d59685811d79ba11529aad5cdf15e84/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f", size = 252791, upload-time = "2026-08-15T08:16:57.103Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3e/48f4cd187b1c33189d86039e9cbe4f92c05454175504b44ff81806d4d1bf/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c", size = 240730, upload-time = "2026-08-15T08:16:58.418Z" }, + { url = "https://files.pythonhosted.org/packages/42/85/f9e22af69af67c54cce42be9455d9c81294f918b4ccc454db01f66efcac2/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18", size = 280791, upload-time = "2026-08-15T08:16:59.918Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4c/9044135f42127630b6fa742feb51256353f6ab87a78f2fdd1de3de955a7f/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5", size = 259598, upload-time = "2026-08-15T08:17:01.421Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ed/1dd7cfebb4e75812934c49ca3b79757d11948053f7937ab7070c151f3c55/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b", size = 278217, upload-time = "2026-08-15T08:17:02.782Z" }, + { url = "https://files.pythonhosted.org/packages/bf/eb/239c84503cc9e3ba6eb34686a24bc66e84f3924efdd7e38e751a19f6bc10/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6", size = 263417, upload-time = "2026-08-15T08:17:04.216Z" }, + { url = "https://files.pythonhosted.org/packages/37/ab/4e4510e1e288478e2c8333131d1c1382382ba8cd2165053c79e39d1da961/charset_normalizer-3.5.1-cp311-cp311-win32.whl", hash = "sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b", size = 181774, upload-time = "2026-08-15T08:17:05.58Z" }, + { url = "https://files.pythonhosted.org/packages/e3/57/32f0ccea59e8612057c61d6fd22ef2cb63cca93c9fe594094919696ac170/charset_normalizer-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9", size = 206653, upload-time = "2026-08-15T08:17:07.075Z" }, + { url = "https://files.pythonhosted.org/packages/17/d4/b65c433fc521e58b5f54293982a5e51c05cb5f2dd3f1c7a6acb65b75324e/charset_normalizer-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10", size = 185630, upload-time = "2026-08-15T08:17:08.502Z" }, + { url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" }, + { url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" }, + { url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" }, + { url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" }, + { url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" }, + { url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" }, + { url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" }, + { url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" }, + { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" }, + { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" }, + { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" }, + { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" }, + { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" }, + { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" }, + { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" }, + { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" }, + { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" }, + { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" }, + { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" }, + { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" }, + { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" }, + { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" }, + { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" }, + { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" }, + { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" }, + { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" }, + { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" }, + { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" }, + { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" }, + { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" }, + { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" }, + { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" }, + { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" }, + { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" }, + { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" }, + { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" }, + { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" }, + { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" }, + { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" }, + { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" }, + { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" }, + { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" }, + { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" }, + { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" }, + { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" }, + { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" }, + { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" }, + { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" }, + { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" }, + { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" }, + { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" }, + { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" }, + { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" }, + { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" }, + { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" }, + { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" }, + { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" }, + { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" }, + { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" }, + { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" }, + { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" }, +] + [[package]] name = "click" version = "8.4.2" @@ -86,6 +263,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/90/fb8f1c84537fbf210c1f53a53ae473a805f6599c5a40b93c1bbadd211f7a/googleapis_common_protos-1.75.2.tar.gz", hash = "sha256:8829a3d1e4508c5b7b9a6b9525f7fccff611f8531644579a76466c29295d4bb2", size = 154083, upload-time = "2026-08-25T19:19:13.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/5b/1c9e55363c3b1890a98cae813de5b4ea327845756cd8fb7ee690140c7eac/googleapis_common_protos-1.75.2-py3-none-any.whl", hash = "sha256:6b83302f554ea93a0f48409c7fc2050f954bcbcddb7e3a9c76d4a823cb22920e", size = 307002, upload-time = "2026-08-25T19:18:08.927Z" }, +] + [[package]] name = "greenlet" version = "3.5.4" @@ -280,6 +469,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/b1/d1f150b2ab3b4ae9932c05104fe1edbcb7fbf505587ea8db99e49341a05f/grpcio_tools-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:b8c9686b0c19f70b63d8d6cfeff5ad3480bdedecd60f14711fe43950f5397253", size = 1224199, upload-time = "2026-07-23T15:22:16.064Z" }, ] +[[package]] +name = "idna" +version = "3.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, +] + [[package]] name = "itsdangerous" version = "2.2.0" @@ -395,6 +593,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/47/80d9e9d468dc5de3af5096f5ccdb065fa4dd1470f74495cc53e59e397f47/opentelemetry_exporter_otlp_proto_grpc-1.44.0.tar.gz", hash = "sha256:40d1ae9e03fcc36de3cbac610cc99f35894938bff9cfd90fc4ec68bd85448463", size = 27225, upload-time = "2026-07-16T15:25:38.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/29/6ae42ba32b153ae0a44ae125f0caff2188bbe62d99c82d1768da30864e72/opentelemetry_exporter_otlp_proto_grpc-1.44.0-py3-none-any.whl", hash = "sha256:6a1a645ea182a2f59440c51fa8301d309f3324a8f9d65f8395584b064b67ee4e", size = 19624, upload-time = "2026-07-16T15:25:19.096Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -606,6 +903,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "s3transfer" version = "0.19.2" @@ -727,6 +1039,10 @@ dependencies = [ { name = "flask" }, { name = "grpcio" }, { name = "grpcio-tools" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "psutil" }, { name = "psycopg", extra = ["binary"] }, { name = "pyyaml" }, @@ -740,6 +1056,10 @@ requires-dist = [ { name = "flask" }, { name = "grpcio" }, { name = "grpcio-tools" }, + { name = "opentelemetry-api", specifier = ">=1.44.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.44.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.44.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.44.0" }, { name = "psutil" }, { name = "psycopg", extras = ["binary"] }, { name = "pyyaml" }, diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 4e416b4..496ae1a 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,6 +3,7 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit +import importlib.util import logging import signal import subprocess @@ -15,6 +16,7 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs +from ventis.controller.utils.process_supervisor import ProcessSupervisor from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -100,6 +102,31 @@ def __init__(self, config_path): self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() + # Spawn the OTLP exporter as a separate process (see OTel_Exporter/DESIGN.md), + # supervised so it gets restarted if it ever exits unexpectedly. + otel_exporter_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "OTel_Exporter", + ) + otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") + self.process_supervisor = ProcessSupervisor() + # `otel:` in global_controller.yaml maps straight to the OTel SDK's own + # standard env vars, not app-specific args -- the exporter subprocess itself + # stays a plain vendor-neutral OTel process; see OTel_Exporter/DESIGN.md. + otel_env = self._otel_exporter_env(self.config.get("otel", {})) + self.process_supervisor.register( + "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + ) + self.process_supervisor.start_all() + + # waiting table GC writes future data into (see OTel_Exporter/db.py); the + # exporter process itself calls init_db() to create the table. + otel_db_spec = importlib.util.spec_from_file_location( + "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") + ) + self._otel_db = importlib.util.module_from_spec(otel_db_spec) + otel_db_spec.loader.exec_module(self._otel_db) + # ------------------------------------------------------------------ # # Stale container cleanup # # ------------------------------------------------------------------ # @@ -143,6 +170,22 @@ def _load_config(config_path): with open(config_path, "r") as f: return yaml.safe_load(f) + @staticmethod + def _otel_exporter_env(otel_cfg): + """Translate global_controller.yaml's `otel:` section into standard OTLP env + vars for the exporter subprocess; returns None if `otel:` is absent/empty so + the subprocess falls back to the SDK's own defaults untouched.""" + env = {} + if otel_cfg.get("protocol"): + env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] + if otel_cfg.get("endpoint"): + env["OTEL_EXPORTER_OTLP_ENDPOINT"] = otel_cfg["endpoint"] + if otel_cfg.get("headers"): + env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( + f"{k}={v}" for k, v in otel_cfg["headers"].items() + ) + return env or None + @staticmethod def _get_replica_placements(ctrl): """Normalize replicas into a list of (host, port) placements.""" @@ -407,14 +450,25 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ + # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which + # terminates every managed process) via the signal handler before this line is + # reached -- without the guard, this could respawn a process just intentionally + # killed. See OTel_Exporter/DESIGN.md. + if self.running: + self.process_supervisor.check_and_respawn() + for instance in self.instance_manager.list_instances(): name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) send_runtime_information( - pull_runtime_information(node_redis), + future_rows, node_redis, self.config.get("database", {}).get("url"), ) @@ -654,6 +708,7 @@ def stop(self): self.running = False self._stop_docker_agents() self._stop_redis_containers() + self.process_supervisor.terminate_all() logger.info("Global controller shut down.") diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py new file mode 100644 index 0000000..8f5724c --- /dev/null +++ b/ventis/controller/utils/process_supervisor.py @@ -0,0 +1,59 @@ +"""Registry for OS processes GlobalController spawns and supervises. + +register() + start_all() spawn processes; check_and_respawn() (call from GC's existing +poll tick) restarts any that exit unexpectedly; terminate_all() (call from GC's shutdown +path) stops them all cleanly. Deliberately GC-agnostic -- callers are responsible for not +calling check_and_respawn() during their own shutdown (see OTel_Exporter/DESIGN.md's +shutdown-race note). +""" + +import logging +import os +import subprocess + +logger = logging.getLogger(__name__) + + +class ProcessSupervisor: + def __init__(self): + self._specs = {} # name -> (argv, env) tuple + self._procs = {} # name -> subprocess.Popen + + def register(self, name, argv, env=None): + """Declare a process to manage. Does not start it -- call start_all() once + everything is registered. `env`, if given, is merged on top of (not a + replacement for) this process's own environment, so the child still inherits + PATH etc.""" + self._specs[name] = (argv, env) + + def start_all(self): + for name, (argv, env) in self._specs.items(): + self._start(name, argv, env) + + def _start(self, name, argv, env=None): + merged_env = {**os.environ, **env} if env else None + self._procs[name] = subprocess.Popen(argv, env=merged_env) + + def check_and_respawn(self): + """Restart any registered process that has exited.""" + for name, proc in list(self._procs.items()): + if proc.poll() is not None: + logger.warning( + "Managed process %r exited (code %s), respawning", + name, + proc.returncode, + ) + argv, env = self._specs[name] + self._start(name, argv, env) + + def terminate_all(self, timeout=10): + """Terminate every managed process, falling back to kill() on timeout.""" + for proc in self._procs.values(): + proc.terminate() + for name, proc in self._procs.items(): + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + self._procs.clear() diff --git a/ventis/deploy.py b/ventis/deploy.py index b6ac721..148d3a4 100644 --- a/ventis/deploy.py +++ b/ventis/deploy.py @@ -9,7 +9,7 @@ import ventis def my_workflow(query: str): - finance = FinanceAgentStub() + finance = FinanceAgent() price = finance.get_stock_price(ticker=query) return {"price": price.value()} diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 8c956ef..d9480be 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -164,13 +164,12 @@ def _build_stub_class(agent_config): Build an AST node for the entire stub class. Generates a class like: - class FinanceAgentStub(object): + class FinanceAgent(object): def __init__(self): pass ...stub methods... """ - # class_name = agent_config["name"] + "Stub" - class_name = agent_config["name"] + class_name = agent_config["name"] functions = agent_config.get("functions", []) # __init__ method: simple pass, no gRPC setup needed. @@ -241,7 +240,7 @@ def generate_stub(yaml_path, output_path): with open(output_path, "w") as f: f.write(source) - class_name = agent_config["name"] + "Stub" + class_name = agent_config["name"] print(f"Generated stub class '{class_name}' -> {output_path}") return source @@ -521,7 +520,7 @@ def start_lc(): "-o", "--output", default=None, - help="Output path for the generated stub file (default: stubs/_stub.py)", + help="Output path for the generated stub file (default: stubs/.py)", ) parser.add_argument( "--agent-file", From 4af43ae3abb8da914050f9f83c61cbf2badebf17 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Tue, 25 Aug 2026 20:40:54 -0700 Subject: [PATCH 06/27] [Feature] Pass env / secrets into agent containers Users had no way to get API keys (OpenAI, Anthropic, embedding models) into an agent container. Add a top-level `env_file` key to global_controller.yaml pointing at a local .env file, which reaches every container as `docker run --env-file`. - resolve_env_file validates the path before anything launches, so a missing .env fails at deploy time instead of deep inside a container. Relative paths resolve against the project root, matching entrypoint. - env_file_args is a context manager owning the local-vs-remote decision and the cleanup, so both runtimes share one code path. Local containers read the original file; remote containers get a copy that is deleted as soon as `docker run` returns, whether or not it succeeded. - GlobalController._push_file streams the file over ssh under `umask 077` rather than scp, so the copy is never briefly world-readable and the secret never lands in a command line. _run_cmd's ssh options moved to a shared _ssh_args. --env-file is appended after the explicit -e VENTIS_* flags; Docker gives those precedence regardless of order, so a stray VENTIS_* line in someone's .env cannot break agent wiring. Closes #50 --- ventis/cli.py | 10 ++ .../cloud_provider_logic/EC2/_runtime.py | 12 ++- .../cloud_provider_logic/Local/_runtime.py | 12 ++- ventis/controller/global_controller.py | 94 +++++++++++++------ ventis/controller/utils/env_file.py | 92 ++++++++++++++++++ 5 files changed, 185 insertions(+), 35 deletions(-) create mode 100644 ventis/controller/utils/env_file.py diff --git a/ventis/cli.py b/ventis/cli.py index b43a6b3..6d85e1f 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -16,6 +16,8 @@ import subprocess import sys +from ventis.controller.utils.env_file import resolve_env_file + logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" @@ -397,6 +399,14 @@ def cmd_deploy(args): config = _load_config(config_path) project_dir = os.getcwd() + # Fail here rather than after a fleet of containers is already up without + # the API keys they need. + try: + resolve_env_file(config, base_dir=project_dir) + except ValueError as e: + logger.error("%s", e) + sys.exit(1) + _ensure_grpc_stubs_importable(project_dir) if any( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 4d5f766..9955fa2 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -23,6 +23,7 @@ import boto3 +from ventis.controller.utils.env_file import env_file_args from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.utils.redis_client import RedisClient @@ -285,8 +286,15 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) if project_id: cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) - cmd.append(image) - result = _controller._run_cmd(cmd, host, user=ssh_user) + + # User secrets from `env_file`. Explicit -e flags above still win over + # anything in the file. + with env_file_args( + _controller, host, ssh_user, container_name, is_local=False + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _controller._run_cmd(cmd, host, user=ssh_user) if result.returncode != 0: raise RuntimeError( f"SSH bootstrap failed on {host}: {(result.stderr or result.stdout or '').strip()}" diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index 963eef3..a387f7b 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -8,6 +8,8 @@ import logging +from ventis.controller.utils.env_file import env_file_args + logger = logging.getLogger(__name__) DEFAULT_HOST = "localhost" @@ -110,9 +112,15 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): cmd.extend(["--memory", f"{resources['memory']}m"]) if resources.get("gpu"): cmd.extend(["--gpus", str(resources["gpu"])]) - cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) + # User secrets from `env_file`. Explicit -e flags above still win, so a + # stray VENTIS_* line in someone's .env cannot break agent wiring. + with env_file_args( + _require_controller(), host, user, runtime_id, _is_local_host(host) + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _require_controller()._run_cmd(cmd, host, user) if result.returncode != 0: raise RuntimeError(f"Failed to launch {runtime_id}") diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 1e24f10..241daff 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -4,6 +4,7 @@ import atexit import logging +import shlex import signal import subprocess import threading @@ -16,6 +17,7 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs +from ventis.controller.utils.env_file import resolve_env_file from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -64,6 +66,9 @@ class GlobalController(object): def __init__(self, config_path): self.config_path = config_path self.config = self._load_config(config_path) + # Validate before launching anything: an agent that boots without its + # API keys fails deep inside a container, where it is expensive to debug. + self.env_file_path = resolve_env_file(self.config) redis_cfg = self.config.get("redis", {}) self.redis = RedisClient( @@ -174,6 +179,7 @@ def reload_config(self): """Reload the config file and rebuild the routing table.""" logger.info("Reloading config from %s", self.config_path) self.config = self._load_config(self.config_path) + self.env_file_path = resolve_env_file(self.config) self.controllers = self.config.get("agents", []) self.poll_interval = self.config.get("poll_interval", 5) assign_project_id(self.config.get("project_id", 0)) @@ -642,6 +648,28 @@ def _send(instance): # Runtime launching # # ------------------------------------------------------------------ # + def _ssh_args(self, host, user=None): + """Return the `ssh ... target` prefix used to reach a remote host.""" + ssh_key_path = os.path.expanduser( + self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/ventis_ec2") + ) + return [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "IdentitiesOnly=yes", + "-o", + "ConnectTimeout=10", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-i", + ssh_key_path, + f"{user}@{host}" if user else host, + ] + def _run_cmd(self, cmd, host, user=None): """ Run a command locally or on a remote host via SSH. @@ -656,41 +684,45 @@ def _run_cmd(self, cmd, host, user=None): """ is_local = _is_local_host(host) if is_local: - return subprocess.run( - cmd, capture_output=True, text=True, timeout=180 - ) - else: - ssh_key_path = os.path.expanduser( - self.config.get("ec2", {}).get( - "ssh_private_key_path", "~/.ssh/ventis_ec2" - ) - ) - ssh_target = f"{user}@{host}" if user else host - remote_cmd = " ".join(cmd) - if cmd and cmd[0] == "docker": - remote_cmd = f"sudo {remote_cmd}" - return subprocess.run( - [ - "ssh", - "-o", - "StrictHostKeyChecking=no", - "-o", - "IdentitiesOnly=yes", - "-o", - "ConnectTimeout=10", - "-o", - "ServerAliveInterval=10", - "-o", - "ServerAliveCountMax=3", - "-i", - ssh_key_path, - ssh_target, - remote_cmd, - ], + return subprocess.run(cmd, capture_output=True, text=True, timeout=180) + + remote_cmd = " ".join(cmd) + if cmd and cmd[0] == "docker": + remote_cmd = f"sudo {remote_cmd}" + return subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + capture_output=True, + text=True, + timeout=180, + ) + + def _push_file(self, local_path, remote_path, host, user=None): + """ + Copy a local file to a remote host over SSH. + + Streams the bytes through `cat` under `umask 077` rather than using + `scp`, so a secrets file is never briefly world-readable on the far + side. + + Returns: + subprocess.CompletedProcess + """ + remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + with open(local_path, "rb") as f: + result = subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + stdin=f, capture_output=True, text=True, timeout=180, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to copy {local_path} to {host}:{remote_path}: " + f"{(result.stderr or result.stdout or '').strip()}" ) + return result def launch_docker_agents(self): """Launch all configured runtimes through InstanceManager.""" diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py new file mode 100644 index 0000000..c83f770 --- /dev/null +++ b/ventis/controller/utils/env_file.py @@ -0,0 +1,92 @@ +""" +Pass user secrets (API keys and friends) into agent containers. + +The user points `env_file` in `config/global_controller.yaml` at a local +`.env` file. Containers on this machine read that file directly; containers +on a remote host get a short-lived 0600 copy. Either way the file reaches +Docker as `--env-file`. +""" + +import logging +import os +from contextlib import contextmanager + +logger = logging.getLogger(__name__) + +REMOTE_ENV_DIR = "/tmp" + + +def resolve_env_file(config, base_dir=None): + """ + Return the absolute path of the configured env file, or None when unset. + + Relative paths resolve against `base_dir` (default: the current working + directory), matching how `entrypoint` and `workflow_file` are resolved. + + Raises: + ValueError: the file is configured but unusable. Deploy should fail + here rather than start a fleet of agents with no API keys. + """ + raw = config.get("env_file") + if not raw: + return None + + path = os.path.expanduser(str(raw)) + if not os.path.isabs(path): + path = os.path.join(base_dir or os.getcwd(), path) + path = os.path.abspath(path) + + if not os.path.exists(path): + raise ValueError(f"env_file does not exist: {path} (from env_file: {raw})") + if not os.path.isfile(path): + raise ValueError(f"env_file is not a file: {path} (from env_file: {raw})") + if not os.access(path, os.R_OK): + raise ValueError(f"env_file is not readable: {path}") + return path + + +def remote_env_path(container_name): + """Where a remote host holds this container's copy of the env file.""" + return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + + +@contextmanager +def env_file_args(controller, host, user, container_name, is_local): + """ + Yield the `docker run` flags that hand the user's env file to a container. + + A container on this machine reads the original file. A container on a + remote host gets a 0600 copy, deleted as soon as the `with` body ends -- + success or failure, since by then the container holds the variables + itself. Keep that body tight around `docker run` so the copy is never + on the host longer than it has to be. + + Yields an empty list when no `env_file` is configured. + """ + env_file_path = getattr(controller, "env_file_path", None) + if not env_file_path: + yield [] + return + + if is_local: + yield ["--env-file", env_file_path] + return + + remote_path = remote_env_path(container_name) + controller._push_file(env_file_path, remote_path, host, user=user) + try: + yield ["--env-file", remote_path] + finally: + _remove_remote_copy(controller, remote_path, host, user) + + +def _remove_remote_copy(controller, remote_path, host, user): + """Delete a remote copy. Best effort -- never masks the caller's error.""" + try: + result = controller._run_cmd(["rm", "-f", remote_path], host, user=user) + if getattr(result, "returncode", 0) != 0: + logger.warning("Failed to delete env file copy %s on %s", remote_path, host) + except Exception as e: + logger.warning( + "Failed to delete env file copy %s on %s: %s", remote_path, host, e + ) From 087ee15b66e967937297580fc551c121c7301a20 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Wed, 26 Aug 2026 13:28:52 -0700 Subject: [PATCH 07/27] Harden the remote env file copy against a hostile /tmp Two holes in the remote staging path, both found reviewing the feature commit. `umask 077` only governs files the shell creates, and `>` follows symlinks -- so it did not actually guarantee a 0600 copy. The destination path is fully predictable (`/tmp/ventis-env-ventis-ec2--`), so a local user on the remote host could pre-create it world-readable, or point it at a file of their own, and collect the API keys. Remove whatever sits at the path before writing; `rm -f` unlinks a symlink rather than following it, so `cat >` then creates a fresh file under the umask. `_run_cmd` joins its argv with spaces and hands the result to a remote shell unquoted. `_push_file` quoted its path but the cleanup `rm` did not, so a container name containing a space split the `rm` into two arguments that matched nothing -- it exited 0 while the secrets file stayed on the host, and the returncode check logged nothing. Scrub the name down to [A-Za-z0-9_.-] in remote_env_path, which also closes the same gap in the `--env-file` argument and in any future use of that path. Still open, tracked separately: a push that dies mid-transfer can leave a copy behind, since the cleanup only covers the `docker run` that follows. On EC2 the instance is terminated on that path, which disposes of it. --- ventis/controller/global_controller.py | 9 ++++++++- ventis/controller/utils/env_file.py | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 241daff..0b30307 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -704,10 +704,17 @@ def _push_file(self, local_path, remote_path, host, user=None): `scp`, so a secrets file is never briefly world-readable on the far side. + Anything already sitting at the destination is removed first: `umask` + only governs files the shell creates, and `>` follows symlinks. Without + the `rm`, a local user on the remote host could pre-create the path + world-readable, or point it at a file of their own, and collect + whatever we write there. + Returns: subprocess.CompletedProcess """ - remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + quoted = shlex.quote(remote_path) + remote_cmd = f"umask 077; rm -f {quoted}; cat > {quoted}" with open(local_path, "rb") as f: result = subprocess.run( self._ssh_args(host, user) + [remote_cmd], diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py index c83f770..6cd77b6 100644 --- a/ventis/controller/utils/env_file.py +++ b/ventis/controller/utils/env_file.py @@ -9,11 +9,13 @@ import logging import os +import re from contextlib import contextmanager logger = logging.getLogger(__name__) REMOTE_ENV_DIR = "/tmp" +_UNSAFE_PATH_CHARS = re.compile(r"[^A-Za-z0-9_.-]") def resolve_env_file(config, base_dir=None): @@ -46,8 +48,17 @@ def resolve_env_file(config, base_dir=None): def remote_env_path(container_name): - """Where a remote host holds this container's copy of the env file.""" - return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + """ + Where a remote host holds this container's copy of the env file. + + The name is scrubbed down to a shell-safe alphabet. This path is + interpolated into remote commands that `_run_cmd` joins with spaces and + hands to a shell unquoted, so a container name carrying a space would + split the cleanup `rm` into two harmless arguments -- it would exit 0 + while the secrets stayed on the host, with nothing in the log to say so. + """ + safe_name = _UNSAFE_PATH_CHARS.sub("-", container_name) + return f"{REMOTE_ENV_DIR}/ventis-env-{safe_name}" @contextmanager From db0ba260c9cc1f88943d98836ce556fb7b458817 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:05:41 -0700 Subject: [PATCH 08/27] WIP: OTel multi-destination fan-out (Railway+Langfuse+Grafana) + cleanup-race fix (pre-pull checkpoint) Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 102 ++++-- OTel_Exporter/convert.py | 11 +- OTel_Exporter/db.py | 44 ++- OTel_Exporter/otel_exporter.py | 266 +++++++++++++-- .../portfolio/config/global_controller.yaml | 18 + pyproject.toml | 5 + tests/test_otel_exporter_fanout.py | 318 ++++++++++++++++++ tests/test_otel_exporter_fields.py | 99 ++++++ uv.lock | 128 +++++++ ventis/controller/global_controller.py | 173 +++++++++- 10 files changed, 1074 insertions(+), 90 deletions(-) create mode 100644 tests/test_otel_exporter_fanout.py create mode 100644 tests/test_otel_exporter_fields.py diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md index 73c4da2..86a2a95 100644 --- a/OTel_Exporter/DESIGN.md +++ b/OTel_Exporter/DESIGN.md @@ -1,6 +1,6 @@ # OTLP Exporter for Ventis GlobalController — Design -Status: **implemented (single-table design)**. `GlobalController` writes futures into a +Status: **implemented (single-table design; multi-destination fan-out in progress)**. `GlobalController` writes futures into a `waiting` table (SQLite); a GC-supervised, GC-restarted OTel Exporter process reads finished/unsent rows, converts each to an OTel span, and hands it to a real `BatchSpanProcessor`/`OTLPSpanExporter`. Batching, serialization, and sending are all @@ -41,7 +41,12 @@ Decisions (final status): originally-planned `database.url` repurposing (below, kept for history) was decided against — env-var configuration is the SDK's own idiomatic mechanism, so no exporter-side config plumbing was added, only a GC-side YAML→env-var translation. - Does not (yet) support simultaneous multi-destination export — see "Known gaps". + The initial multi-destination extension uses one `otel.destinations` list and one + independent exporter/`BatchSpanProcessor` pair per destination. gRPC and HTTP + destinations may be mixed in the same list. The legacy single-destination fields + remain supported through the original standard-environment-variable path. + Configuration is read at exporter startup; changing it requires a + GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing @@ -64,39 +69,47 @@ Decisions (final status): `global_controller.yaml` gains an optional `otel:` section: ```yaml otel: - protocol: grpc # or http - endpoint: otlp-pg-receiver.railway.internal:4317 - headers: {} # e.g. Authorization: "Basic " for a backend needing auth + destinations: + - name: railway + protocol: grpc # or http + endpoint: otlp-pg-receiver.railway.internal:4317 + headers: {} + - name: langfuse + protocol: http + endpoint: https://cloud.langfuse.com/api/public/otel/v1/traces + headers: {} # e.g. Authorization: "Basic " ``` -`GlobalController._otel_exporter_env()` translates this into -`OTEL_EXPORTER_OTLP_PROTOCOL`/`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` -and hands them to `ProcessSupervisor.register("otel_exporter", ..., env=...)`, which now -supports an `env` param (merged on top of the parent process's own environment, not a -replacement). Omitting `otel:` entirely falls back to whatever ambient env the exporter -subprocess would otherwise inherit, same as before this change. +`GlobalController._otel_exporter_env()` translates each destination into the exporter +process's destination configuration and hands it to `ProcessSupervisor.register( +"otel_exporter", ..., env=...)`, which supports an `env` param (merged on top of the +parent process's own environment, not a replacement). The legacy single-destination +`protocol`/`endpoint`/`headers` form remains valid and continues through the SDK's +standard OTLP environment variables. Omitting `otel:` entirely falls back to whatever +ambient env the exporter subprocess would otherwise inherit, same as before this +change. -`otel_exporter.py` itself reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly (to pick -which `OTLPSpanExporter` class to import — gRPC or HTTP; the plain SDK classes don't -self-select this the way `opentelemetry-instrument`'s auto-config does). Endpoint and -headers are never read directly — `OTLPSpanExporter()` is still constructed with no -explicit args, letting the SDK resolve those from the same env vars itself, exactly as -before this change. `BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000)` +`otel_exporter.py` parses the destination configuration at startup and constructs the +appropriate OTLP exporter for each entry (gRPC or HTTP), passing that destination's +endpoint and headers to the SDK. `BatchSpanProcessor(..., schedule_delay_millis=1000)` — the flush delay is explicitly overridden from the SDK default (5000ms) to 1000ms; `max_export_batch_size` is left at the SDK default (512), which already approximates the original "500 spans" batching ask without any override needed. ### 2. `OTel_Exporter/otel_exporter.py` A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM -stays responsive), calling `_send_pending()` each tick: +stays responsive), calling `_send_pending()` each tick. At startup it constructs one +independent OTLP exporter and `BatchSpanProcessor` for each configured destination; +each pair may use a different protocol, endpoint, and headers: - `SELECT * FROM waiting WHERE finished_at IS NOT NULL AND (sent IS NULL OR sent = 0)`. - Per row, each isolated in its own try/except (one malformed row is logged and skipped, never blocks the rest of the batch): `convert.waiting_row_to_span(row)` → - `_processor.on_end(span)` → `db.mark_sent(future_id)` immediately — atomic per row, not - batched at the end, so a crash mid-poll can't leave an already-sent row unmarked (which - would cause a duplicate send on the next run). -- `_processor` is constructed once at startup; no `TracerProvider` is used at all, since - spans are hand-built and handed straight to the processor via `on_end()`. -- `_processor.shutdown()` on exit, flushing any pending batch. + `on_end(span)` on every configured processor → `db.mark_sent(future_id)` immediately. + The row is marked after it has been queued to all processors. `sent` therefore means + **queued to every configured destination**, not remotely acknowledged; this is the + initial best-effort delivery contract and retains the existing single boolean schema. +- Each processor is constructed once at startup; no `TracerProvider` is used at all, + since spans are hand-built and handed straight to the processors via `on_end()`. +- Every processor is shut down on exit, flushing its pending batch independently. ### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) `future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is @@ -117,7 +130,11 @@ no live exception object, only strings) plus `Status(StatusCode.ERROR, descripti **Attribute naming**: `model`/`input_token_count`/`output_token_count` are set under the real, current OTel GenAI semantic-convention keys — `gen_ai.request.model`/ `gen_ai.usage.input_tokens`/`gen_ai.usage.output_tokens` — verified against the actual -spec (`open-telemetry/semantic-conventions`), not assumed. `cpu`/`gpu`/ +spec (`open-telemetry/semantic-conventions`), not assumed. Submitted `args` and the +completed `result` are stored in `waiting.input`/`waiting.output` as valid JSON text and +exported under Langfuse's documented `langfuse.observation.input`/ +`langfuse.observation.output` attributes. The span name is the stable logical +`service.method`, not the executing instance's UUID. `cpu`/`gpu`/ `execution_time_ms`/`queue_time_ms`/`token_count` keep plain names deliberately: none of them have an OTel GenAI equivalent (cpu/gpu/queue-time are Ventis infra concepts, and `token_count`, an input+output sum, isn't part of the spec at all — inventing a @@ -136,24 +153,45 @@ process (all `.terminate()` calls first, then `.wait()` on each, falling back to `.kill()`), called from GC's `stop()`. Adding a future second daemon is one more `register()` call — no new spawn/monitor/terminate code needed. -### 5. Dependencies (all added) +### 5. Poll/cleanup race fix (`ventis/controller/global_controller.py`) +GC's cleanup thread used to run on its own `cleanup_interval` timer (default 10s), +fully independent of the poll loop's `poll_interval` (default 5s) that writes futures +into `waiting`. On a fast-completing request, cleanup could delete a session's Redis +future keys before the next poll tick ever read them, so those futures never reached +`waiting` at all — silently dropped from every OTel destination, not just one. +Reproduced live: a fast request left only 1 of 6 agent calls in `waiting`. Fixed by +having the poll loop signal a `threading.Event` (`_cleanup_ready`) right after each +tick; the cleanup thread waits on that event instead of sleeping on its own timer, so +cleanup only ever runs immediately after a poll has already captured that tick's state. +Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is a +fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall +the poll loop's health checks and OTel writes. + +### 6. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). ## Known gaps (not yet built) +- `WriteResult()` passes an undefined `error_message` variable to its fan-out callback, + which can interrupt remote consumer propagation after the callback hash is persisted. +- Redis records failure text under `error`, but the waiting-table writer reads + `error_name`/`error_message`, so exported exception details are usually empty. +- Rows are marked `sent` immediately after `BatchSpanProcessor.on_end()` accepts them, + before the asynchronous OTLP export is confirmed; a later delivery failure can lose a + span while leaving `sent = 1`. - Spans carry no explicit `resource`/`instrumentation_scope` — would show as `service.name=unknown_service` at a real backend. -- No simultaneous multi-destination export — `otel:` configures exactly one - destination; sending to two backends at once would mean registering a second, - separately-configured `otel_exporter` subprocess (same script, different env), not - something the exporter or its config format do today. +- Destination-specific delivery acknowledgement/retry state is not tracked yet: + `sent` only records that the span was queued to all configured processors, so an + asynchronous export failure can still lose a span until a later delivery-state design + is added. - `waiting` grows unboundedly: sent rows are never pruned, and futures that never finish (`finished_at` never arrives) also stay forever, invisible and un-expiring. - `error_name` is always `NULL` — Ventis's own Redis writer never records a distinct exception-type field, only a message string. -- No committed test suite — all verification during development was ad hoc scripts, not - `pytest` files under `tests/`. +- Test coverage is still limited; the waiting-field migration/normalization/conversion + path is covered, but the exporter process and live OTLP delivery are not. - Never verified against a live OTLP receiver — only against a refused connection (confirmed the SDK's real retry/error-handling path is exercised correctly). - No retry-limit/quarantine for a permanently malformed row — it logs an error every poll diff --git a/OTel_Exporter/convert.py b/OTel_Exporter/convert.py index 8e062a4..b7fb493 100644 --- a/OTel_Exporter/convert.py +++ b/OTel_Exporter/convert.py @@ -64,10 +64,9 @@ def waiting_row_to_span(row): ) status = Status(StatusCode.ERROR, description=row.get("error_message")) - # model/input/output use real OTel GenAI semconv names; cpu/gpu/execution_time_ms/ - # queue_time_ms/token_count have no semconv equivalent (Ventis infra concepts, or -- - # for token_count -- a derived sum the spec doesn't define), so they keep plain names - # rather than being forced into a fake gen_ai.* one. See DESIGN.md. + # Model and token usage use OTel GenAI semantic-convention names. Observation + # input/output use Langfuse's documented JSON-string attributes. The remaining + # Ventis infrastructure values have no GenAI equivalent, so they keep plain names. attributes = { k: v for k, v in { @@ -79,12 +78,14 @@ def waiting_row_to_span(row): "gen_ai.usage.input_tokens": row.get("input_token_count"), "gen_ai.usage.output_tokens": row.get("output_token_count"), "token_count": row.get("token_count"), + "langfuse.observation.input": row.get("input"), + "langfuse.observation.output": row.get("output"), }.items() if v is not None } return ReadableSpan( - name=row.get("agent_id") or "unknown_agent", + name=row.get("name") or row.get("agent_id") or "unknown_agent", context=context, parent=parent, attributes=attributes, diff --git a/OTel_Exporter/db.py b/OTel_Exporter/db.py index 4bb301d..a8a675f 100644 --- a/OTel_Exporter/db.py +++ b/OTel_Exporter/db.py @@ -8,6 +8,7 @@ wasn't doing anything BatchSpanProcessor doesn't already do -- see DESIGN.md.) """ +import json import os import sqlite3 @@ -51,16 +52,31 @@ cache_hit_ratio REAL, error_name TEXT, error_message TEXT, + name TEXT, + input TEXT, + output TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, sent BOOLEAN DEFAULT 0 """ +_MIGRATION_COLUMNS = { + "name": "TEXT", + "input": "TEXT", + "output": "TEXT", +} + def init_db(db_path=DB_PATH): - """Create the waiting table if it doesn't already exist.""" + """Create the waiting table and add columns missing from older databases.""" conn = sqlite3.connect(db_path) try: conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") + existing_columns = { + row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() + } + for column, column_type in _MIGRATION_COLUMNS.items(): + if column not in existing_columns: + conn.execute(f"ALTER TABLE waiting ADD COLUMN {column} {column_type}") conn.commit() finally: conn.close() @@ -74,6 +90,7 @@ def init_db(db_path=DB_PATH): "input_token_count", "output_token_count", "token_count", "errors", "failed", "server_cost", "token_cost", "total_cost", "cached_tokens", "cache_hit_ratio", "error_name", "error_message", + "name", "input", "output", ] _WAITING_UPSERT = """ @@ -86,6 +103,17 @@ def init_db(db_path=DB_PATH): ) +def _normalize_json_text(value): + """Return JSON text, encoding legacy scalar strings that are not valid JSON.""" + if value is None or value == "": + return None + try: + json.loads(value) + except (json.JSONDecodeError, TypeError): + return json.dumps(value) + return value + + def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH): """Upsert future rows (as returned by telemetry_logging.pull_runtime_information) into the waiting table. Unlike runtime_information, rows without finished_at are @@ -113,6 +141,17 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH output_token_count = int(float(raw.get("output_token_count") or 0)) token_count = int(float(raw.get("token_count") or 0)) cached_tokens = int(float(raw.get("input_cache_tokens") or 0)) + service = raw.get("service") + method = raw.get("method") + name = raw.get("name") or ".".join( + part for part in (service, method) if part + ) + result = raw.get("result") + # Compatibility with pre-consolidation deployments, where completion + # metrics live in future:{id}:metrics but result lives in future:{id}. + # Unified hashes already include result and avoid this extra read. + if not result and finished_at and redis_client is not None: + result = redis_client.hget(f"future:{fid}", "result") token_cost = ( pricing.compute_token_cost( @@ -165,6 +204,9 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, "error_name": raw.get("error_name"), "error_message": raw.get("error_message"), + "name": name or agent_id or "unknown_agent", + "input": _normalize_json_text(raw.get("args")), + "output": _normalize_json_text(result), }, ) conn.commit() diff --git a/OTel_Exporter/otel_exporter.py b/OTel_Exporter/otel_exporter.py index b1c7f1a..cad2af8 100644 --- a/OTel_Exporter/otel_exporter.py +++ b/OTel_Exporter/otel_exporter.py @@ -1,33 +1,31 @@ -"""Entrypoint for the OTLP Exporter process. - -Each poll tick: read finished, not-yet-sent rows from `waiting`, convert each to a span, -hand it to a BatchSpanProcessor/OTLPSpanExporter, and mark it sent -- batching, OTLP -serialization, and sending are all the SDK's own code, not ours (see DESIGN.md). Each -row's send-and-mark-sent is atomic and happens immediately after its own successful -send, not batched at the end, so a crash mid-poll can't leave an already-sent row -unmarked (which would cause a duplicate send next run). `OTLPSpanExporter()` takes no -explicit endpoint/headers here -- it falls back to the SDK's own standard -`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` env vars, or localhost:4317, -per the SDK's own default behavior. GlobalController sets those env vars (plus -`OTEL_EXPORTER_OTLP_PROTOCOL`, which this module reads itself below to pick the gRPC vs -HTTP class) from `global_controller.yaml`'s `otel:` section when it spawns this process; -this file has no YAML/app-config awareness of its own, only standard OTel env vars -- -see DESIGN.md. +"""Entrypoint for the OTLP exporter process. + +Each poll tick reads finished, not-yet-sent rows from ``waiting``, converts each to a +span, hands it to every configured BatchSpanProcessor, and marks it sent only after +all processors accept it. Batching, OTLP serialization, and sending remain the SDK's +responsibility (see DESIGN.md). + +GlobalController may provide a JSON list in ``VENTIS_OTEL_DESTINATIONS``. That is a +Ventis-specific configuration because the standard OTEL exporter environment +variables describe only one destination. If it is absent, the original single +destination behavior is retained: the exporter class and its settings are selected +from the standard OTEL environment variables and SDK defaults. """ +import json import logging +import math import os import signal import sqlite3 import time -# Protocol is the one thing the SDK's own exporter classes don't self-select from -# OTEL_EXPORTER_OTLP_PROTOCOL -- endpoint/headers/auth stay fully env-var-driven via -# each class's own defaults; see OTel_Exporter/DESIGN.md. -if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").startswith("http"): - from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -else: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as GrpcOTLPSpanExporter, +) +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter as HttpOTLPSpanExporter, +) from opentelemetry.sdk.trace.export import BatchSpanProcessor import convert @@ -38,7 +36,165 @@ _running = True _processor = None +_processors = [] POLL_INTERVAL_SECONDS = 5 +DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" + + +def _normalize_protocol(protocol): + """Return the exporter family for a configured protocol name.""" + if not isinstance(protocol, str) or not protocol.strip(): + raise ValueError("destination protocol must be a non-empty string") + normalized = protocol.strip().lower().replace("_", "-") + if normalized in {"grpc", "otlp/grpc", "grpc/protobuf", "grpc-protobuf"}: + return "grpc" + if normalized in { + "http", + "http/protobuf", + "http-protobuf", + "http/proto", + "http+protobuf", + "protobuf", + }: + return "http" + raise ValueError( + f"unsupported destination protocol {protocol!r}; expected grpc or http/protobuf" + ) + + +def _validate_destination(destination, index): + if not isinstance(destination, dict): + raise ValueError(f"destination {index} must be an object") + + name = destination.get("name") + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"destination {index} name must be a non-empty string") + + protocol = _normalize_protocol(destination.get("protocol")) + endpoint = destination.get("endpoint") + if not isinstance(endpoint, str) or not endpoint.strip(): + raise ValueError(f"destination {name!r} endpoint must be a non-empty string") + + headers = destination.get("headers") + if headers is not None: + if not isinstance(headers, dict): + raise ValueError(f"destination {name!r} headers must be an object") + if any( + not isinstance(key, str) + or not key.strip() + or not isinstance(value, str) + for key, value in headers.items() + ): + raise ValueError( + f"destination {name!r} headers must map non-empty strings to strings" + ) + headers = dict(headers) + + insecure = destination.get("insecure") + if insecure is not None and not isinstance(insecure, bool): + raise ValueError(f"destination {name!r} insecure must be a boolean") + + timeout = destination.get("timeout") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ValueError(f"destination {name!r} timeout must be a positive number") + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError(f"destination {name!r} timeout must be a positive number") + + return { + "name": name.strip(), + "protocol": protocol, + "endpoint": endpoint.strip(), + "headers": headers, + "insecure": insecure, + "timeout": timeout, + } + + +def _configured_destinations(): + """Parse and validate the Ventis multi-destination environment variable. + + ``None`` means no Ventis-specific configuration was supplied, so callers can + preserve legacy OTEL environment-variable behavior. An empty or malformed value + is an explicit configuration error and fails startup rather than silently + exporting to the wrong destination. + """ + raw = os.environ.get(DESTINATIONS_ENV) + if raw is None: + return None + try: + destinations = json.loads(raw) + except (TypeError, json.JSONDecodeError) as exc: + raise ValueError(f"{DESTINATIONS_ENV} must contain a JSON list") from exc + if not isinstance(destinations, list) or not destinations: + raise ValueError(f"{DESTINATIONS_ENV} must contain a non-empty JSON list") + + validated = [] + names = set() + for index, destination in enumerate(destinations): + validated_destination = _validate_destination(destination, index) + name = validated_destination["name"] + if name in names: + raise ValueError(f"destination names must be unique; duplicate {name!r}") + names.add(name) + validated.append(validated_destination) + return validated + + +def _build_exporter(destination): + """Construct one explicitly configured exporter without logging credentials.""" + kwargs = { + "endpoint": destination["endpoint"], + } + if destination["headers"] is not None: + kwargs["headers"] = destination["headers"] + if destination["timeout"] is not None: + kwargs["timeout"] = destination["timeout"] + + if destination["protocol"] == "grpc": + if destination["insecure"] is not None: + kwargs["insecure"] = destination["insecure"] + return GrpcOTLPSpanExporter(**kwargs) + + if destination["insecure"] is not None: + logger.warning( + "Destination %s specifies insecure=%s, which is ignored for HTTP exporters.", + destination["name"], + destination["insecure"], + ) + return HttpOTLPSpanExporter(**kwargs) + + +def _build_processors(): + """Build destination processors, or one legacy processor when unconfigured.""" + destinations = _configured_destinations() + if destinations is None: + protocol = os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower() + exporter_class = ( + HttpOTLPSpanExporter if protocol.startswith("http") else GrpcOTLPSpanExporter + ) + return [("legacy", BatchSpanProcessor(exporter_class(), schedule_delay_millis=1000))] + + processors = [] + try: + for destination in destinations: + exporter = _build_exporter(destination) + processors.append( + ( + destination["name"], + BatchSpanProcessor(exporter, schedule_delay_millis=1000), + ) + ) + logger.info( + "Configured OTel destination %s (%s).", + destination["name"], + destination["protocol"], + ) + except Exception: + for _, processor in processors: + processor.shutdown() + raise + return processors def _handle_shutdown(signum, frame): @@ -48,6 +204,14 @@ def _handle_shutdown(signum, frame): def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" + processors = _processors + if not processors and _processor is not None: + # Compatibility for callers that configured the pre-fan-out singular + # ``_processor`` directly (the normal startup path always populates both). + processors = [("legacy", _processor)] + if not processors: + raise RuntimeError("OTel exporter has no configured processors") + conn = sqlite3.connect(db.DB_PATH) conn.row_factory = sqlite3.Row try: @@ -63,35 +227,63 @@ def _send_pending(): for row in rows: try: span = convert.waiting_row_to_span(row) - _processor.on_end(span) except Exception as e: logger.error( - "Skipping waiting row %s -- failed to send: %s", row["future_id"], e + "Skipping waiting row %s -- failed to convert: %s", row["future_id"], e ) continue + + failed_destinations = [] + for destination_name, processor in processors: + try: + processor.on_end(span) + except Exception as e: + # Still offer the span to the remaining processors. The row is only + # acknowledged when every destination accepted it, so a failed + # destination will be retried by the next poll. + failed_destinations.append(destination_name) + logger.error( + "Destination %s rejected waiting row %s: %s", + destination_name, + row["future_id"], + e, + ) + if failed_destinations: + continue db.mark_sent(row["future_id"]) sent_count += 1 - logger.info("Sent %d span(s) to the batch processor.", sent_count) + logger.info("Queued %d span(s) for all configured OTel destinations.", sent_count) def main(): - global _processor + global _processor, _processors signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _processor = BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000) - logger.info("OTel exporter process started.") - last_poll = 0 - while _running: - if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + _processors = _build_processors() + # Keep the old singular module variable available to integrations that imported + # it, while all sending uses the destination-aware collection above. + _processor = _processors[0][1] + logger.info("OTel exporter process started with %d destination(s).", len(_processors)) + try: + last_poll = 0 + while _running: + if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + try: + _send_pending() + except Exception as e: + logger.warning("Poll cycle failed (non-fatal): %s", e) + last_poll = time.time() + time.sleep(1) + finally: + for destination_name, processor in _processors: try: - _send_pending() + processor.shutdown() except Exception as e: - logger.warning("Poll cycle failed (non-fatal): %s", e) - last_poll = time.time() - time.sleep(1) - _processor.shutdown() - logger.info("OTel exporter process exiting.") + logger.error( + "Failed to shut down OTel destination %s: %s", destination_name, e + ) + logger.info("OTel exporter process exiting.") if __name__ == "__main__": diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index d61ad42..ab5af4d 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -78,6 +78,24 @@ agents: provider: EC2 instance_type: t3.micro + +otel: + destinations: + - name: railway + protocol: grpc + endpoint: yamanote.proxy.rlwy.net:19803 + insecure: true + headers: {} + - name: langfuse + protocol: http + endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces + headers: {} + - name: grafana + protocol: http + endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces + headers: + Authorization: Basic ${GRAFANA_OTLP_HEADERS} + # Polling interval in seconds poll_interval: 5 diff --git a/pyproject.toml b/pyproject.toml index 8410b24..9ae1234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,3 +58,8 @@ allowed-unresolved-imports = [ "*_stub", "*_agent_stub", ] + +[dependency-groups] +dev = [ + "pytest>=9.1.1", +] diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py new file mode 100644 index 0000000..ed1b423 --- /dev/null +++ b/tests/test_otel_exporter_fanout.py @@ -0,0 +1,318 @@ +"""Focused tests for the Ventis OTel exporter fan-out configuration.""" + +import json +import os +import sqlite3 +import sys +import tempfile +import types +import unittest +from unittest.mock import MagicMock, patch + + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +# ``otel_exporter.py`` is also executed as a script from its own directory and +# therefore imports ``convert`` and ``db`` as top-level modules. +sys.path.insert(0, os.path.join(ROOT, "OTel_Exporter")) + +import db # noqa: E402 +import otel_exporter # noqa: E402 + + +# The generated local-controller protobuf modules are build artifacts and are +# not present in a source checkout. The static config helper does not use them, +# so provide the tiny import-time surface needed to test it in isolation. +if "local_controler_pb2" not in sys.modules: + local_pb2 = types.ModuleType("local_controler_pb2") + local_pb2.JsonResponse = object + sys.modules["local_controler_pb2"] = local_pb2 +if "local_controler_pb2_grpc" not in sys.modules: + local_pb2_grpc = types.ModuleType("local_controler_pb2_grpc") + local_pb2_grpc.LocalControllerStub = object + sys.modules["local_controler_pb2_grpc"] = local_pb2_grpc + + +class OTelExporterFanoutTests(unittest.TestCase): + def setUp(self): + self.db_file = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + self.db_path = self.db_file.name + self.db_file.close() + db.init_db(self.db_path) + + def tearDown(self): + os.unlink(self.db_path) + + @staticmethod + def _destination_config(): + return [ + { + "name": "railway", + "protocol": "grpc", + "endpoint": "receiver.example:4317", + "headers": {"x-api-key": "railway-key"}, + "insecure": True, + "timeout": 3.5, + }, + { + "name": "langfuse", + "protocol": "http/protobuf", + "endpoint": "https://langfuse.example/api/public/otel", + "headers": {"authorization": "Basic secret"}, + "timeout": 7, + }, + ] + + def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): + grpc_exporter = object() + http_exporter = object() + grpc_processor = MagicMock(name="grpc_processor") + http_processor = MagicMock(name="http_processor") + destinations = self._destination_config() + + with patch.dict( + os.environ, + {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, + clear=True, + ), patch.object( + otel_exporter, + "GrpcOTLPSpanExporter", + return_value=grpc_exporter, + ) as grpc_constructor, patch.object( + otel_exporter, + "HttpOTLPSpanExporter", + return_value=http_exporter, + ) as http_constructor, patch.object( + otel_exporter, + "BatchSpanProcessor", + side_effect=[grpc_processor, http_processor], + ) as processor_constructor: + processors = otel_exporter._build_processors() + + self.assertEqual( + processors, [("railway", grpc_processor), ("langfuse", http_processor)] + ) + grpc_constructor.assert_called_once_with( + endpoint="receiver.example:4317", + headers={"x-api-key": "railway-key"}, + timeout=3.5, + insecure=True, + ) + http_constructor.assert_called_once_with( + endpoint="https://langfuse.example/api/public/otel", + headers={"authorization": "Basic secret"}, + timeout=7, + ) + self.assertEqual( + processor_constructor.call_args_list, + [ + unittest.mock.call(grpc_exporter, schedule_delay_millis=1000), + unittest.mock.call(http_exporter, schedule_delay_millis=1000), + ], + ) + + def test_build_processors_preserves_legacy_single_destination_fallback(self): + http_exporter = object() + processor = MagicMock(name="legacy_processor") + with patch.dict( + os.environ, + {"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, + clear=True, + ), patch.object( + otel_exporter, "HttpOTLPSpanExporter", return_value=http_exporter + ) as constructor, patch.object( + otel_exporter, "BatchSpanProcessor", return_value=processor + ): + result = otel_exporter._build_processors() + + self.assertEqual(result, [("legacy", processor)]) + constructor.assert_called_once_with() + + def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): + invalid_values = [ + "not-json", + json.dumps([]), + json.dumps( + [ + { + "name": "same", + "protocol": "grpc", + "endpoint": "one:4317", + }, + { + "name": "same", + "protocol": "http/protobuf", + "endpoint": "https://two", + }, + ] + ), + ] + for raw in invalid_values: + with self.subTest(raw=raw), patch.dict( + os.environ, {otel_exporter.DESTINATIONS_ENV: raw}, clear=True + ): + with self.assertRaises(ValueError): + otel_exporter._configured_destinations() + + def test_controller_expands_env_and_builds_langfuse_basic_auth(self): + from ventis.controller.global_controller import GlobalController + + with patch.dict( + os.environ, + { + "LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com", + "LANGFUSE_PUBLIC_KEY": "public", + "LANGFUSE_SECRET_KEY": "secret", + }, + clear=True, + ): + env = GlobalController._otel_exporter_env( + { + "destinations": [ + { + "name": "langfuse", + "protocol": "http/protobuf", + "endpoint": "${LANGFUSE_BASE_URL}/api/public/otel/v1/traces", + } + ] + } + ) + + destination = json.loads(env[otel_exporter.DESTINATIONS_ENV])[0] + self.assertEqual( + destination["endpoint"], + "https://us.cloud.langfuse.com/api/public/otel/v1/traces", + ) + self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") + + def test_controller_env_serializes_destinations_and_keeps_legacy_mapping(self): + # Importing the controller is intentionally local: this test remains + # runnable in the exporter-only environment used by the focused suite. + from ventis.controller.global_controller import GlobalController + + destinations = self._destination_config() + env = GlobalController._otel_exporter_env( + { + "protocol": "grpc", + "endpoint": "legacy.example:4317", + "headers": {"x-tenant": "demo"}, + "destinations": destinations, + } + ) + self.assertEqual(env["OTEL_EXPORTER_OTLP_PROTOCOL"], "grpc") + self.assertEqual(env["OTEL_EXPORTER_OTLP_ENDPOINT"], "legacy.example:4317") + self.assertEqual(env["OTEL_EXPORTER_OTLP_HEADERS"], "x-tenant=demo") + self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) + + def test_controller_rejects_invalid_destinations_before_starting_child(self): + from ventis.controller.global_controller import GlobalController + + invalid_destinations = [ + [], + [{"name": "railway", "protocol": "grpc"}], + [ + {"name": "same", "protocol": "grpc", "endpoint": "one:4317"}, + { + "name": "same", + "protocol": "http/protobuf", + "endpoint": "https://two", + }, + ], + [{"name": "bad", "protocol": "smtp", "endpoint": "example"}], + ] + for destinations in invalid_destinations: + with self.subTest(destinations=destinations), self.assertRaises(ValueError): + GlobalController._otel_exporter_env( + {"destinations": destinations} + ) + + def _insert_pending_row(self): + conn = sqlite3.connect(self.db_path) + try: + conn.execute( + """ + INSERT INTO waiting ( + future_id, session_id, started_at, finished_at, failed, + name, input, output, sent + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) + """, + ( + "00112233445566778899aabbccddeeff", + "ffeeddccbbaa99887766554433221100", + 1.0, + 2.0, + 0, + "PriceAgent.get_history", + '{"ticker":"NVDA"}', + '{"price":100}', + ), + ) + conn.commit() + finally: + conn.close() + + def test_send_pending_delivers_the_same_span_to_every_processor(self): + self._insert_pending_row() + first = MagicMock(name="first") + second = MagicMock(name="second") + with patch.object(otel_exporter.db, "DB_PATH", self.db_path), patch.object( + otel_exporter.db, "mark_sent" + ) as mark_sent: + otel_exporter._processors = [("railway", first), ("langfuse", second)] + otel_exporter._processor = None + otel_exporter._send_pending() + + first.on_end.assert_called_once() + second.on_end.assert_called_once() + self.assertIs(first.on_end.call_args.args[0], second.on_end.call_args.args[0]) + mark_sent.assert_called_once_with("00112233445566778899aabbccddeeff") + + def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_failure(self): + self._insert_pending_row() + failed = MagicMock(name="failed") + failed.on_end.side_effect = RuntimeError("destination unavailable") + remaining = MagicMock(name="remaining") + with patch.object(otel_exporter.db, "DB_PATH", self.db_path), patch.object( + otel_exporter.db, "mark_sent" + ) as mark_sent: + otel_exporter._processors = [("railway", failed), ("langfuse", remaining)] + otel_exporter._processor = None + otel_exporter._send_pending() + + failed.on_end.assert_called_once() + remaining.on_end.assert_called_once() + mark_sent.assert_not_called() + + conn = sqlite3.connect(self.db_path) + try: + self.assertEqual(conn.execute("SELECT sent FROM waiting").fetchone()[0], 0) + finally: + conn.close() + + def test_processor_construction_failure_shuts_down_already_built_processors(self): + first_processor = MagicMock(name="first_processor") + destinations = self._destination_config() + with patch.dict( + os.environ, + {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, + clear=True, + ), patch.object( + otel_exporter, + "GrpcOTLPSpanExporter", + return_value=object(), + ), patch.object( + otel_exporter, + "HttpOTLPSpanExporter", + side_effect=RuntimeError("bad HTTP exporter"), + ), patch.object( + otel_exporter, + "BatchSpanProcessor", + return_value=first_processor, + ): + with self.assertRaisesRegex(RuntimeError, "bad HTTP exporter"): + otel_exporter._build_processors() + + first_processor.shutdown.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py new file mode 100644 index 0000000..4a62c58 --- /dev/null +++ b/tests/test_otel_exporter_fields.py @@ -0,0 +1,99 @@ +import json +import os +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +from OTel_Exporter import convert, db + + +class OTelExporterFieldTests(unittest.TestCase): + def setUp(self): + handle = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + self.db_path = handle.name + handle.close() + + def tearDown(self): + os.unlink(self.db_path) + + def test_init_db_migrates_existing_waiting_table(self): + with sqlite3.connect(self.db_path) as conn: + conn.execute( + "CREATE TABLE waiting (future_id TEXT PRIMARY KEY, session_id TEXT NOT NULL)" + ) + + db.init_db(self.db_path) + + with sqlite3.connect(self.db_path) as conn: + columns = { + row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() + } + self.assertTrue({"name", "input", "output"}.issubset(columns)) + + def test_fields_are_normalized_and_added_to_span(self): + db.init_db(self.db_path) + raw = { + "future_id": "00112233445566778899aabbccddeeff", + "request_id": "ffeeddccbbaa99887766554433221100", + "service": "PriceAgent", + "method": "get_history", + "args": '{"ticker": "NVDA"}', + "result": "plain text result", + "created_at": "1.0", + "finished_at": "2.0", + "failed": "0", + } + + with patch.object(db.pricing, "compute_token_cost", return_value=0.0): + db.write_waiting_rows([raw], db_path=self.db_path) + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM waiting").fetchone() + + self.assertEqual(row["name"], "PriceAgent.get_history") + self.assertEqual(row["input"], raw["args"]) + self.assertEqual(json.loads(row["output"]), raw["result"]) + + span = convert.waiting_row_to_span(row) + self.assertEqual(span.name, "PriceAgent.get_history") + self.assertEqual(span.attributes["langfuse.observation.input"], raw["args"]) + self.assertEqual( + span.attributes["langfuse.observation.output"], row["output"] + ) + + def test_split_hash_result_is_loaded_for_finished_rows(self): + class SplitHashRedis: + def hget(self, key, field): + self.request = (key, field) + return '{"recommendation": "hold"}' + + def get(self, key): + return None + + db.init_db(self.db_path) + redis = SplitHashRedis() + raw = { + "future_id": "11112222333344445555666677778888", + "request_id": "88887777666655554444333322221111", + "service": "AdvisorAgent", + "method": "summarize", + "args": '{"risk": "moderate"}', + "result": "", + "created_at": "1.0", + "finished_at": "2.0", + "failed": "0", + } + + with patch.object(db.pricing, "compute_token_cost", return_value=0.0): + db.write_waiting_rows([raw], redis_client=redis, db_path=self.db_path) + + with sqlite3.connect(self.db_path) as conn: + output = conn.execute("SELECT output FROM waiting").fetchone()[0] + self.assertEqual(redis.request, (f"future:{raw['future_id']}", "result")) + self.assertEqual(json.loads(output), {"recommendation": "hold"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index ba10707..9b86805 100644 --- a/uv.lock +++ b/uv.lock @@ -246,6 +246,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + [[package]] name = "flask" version = "3.1.3" @@ -478,6 +490,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "itsdangerous" version = "2.2.0" @@ -692,6 +713,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -815,6 +854,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1003,6 +1069,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, ] +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" @@ -1050,6 +1170,11 @@ dependencies = [ { name = "sqlalchemy" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "boto3" }, @@ -1067,6 +1192,9 @@ requires-dist = [ { name = "sqlalchemy" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.1.1" }] + [[package]] name = "werkzeug" version = "3.1.8" diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index b84614b..9947680 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,15 +3,18 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit +import base64 import importlib.util +import json import logging +import math +import os +import re import signal import subprocess +import sys import threading import time -import json -import sys -import os from concurrent.futures import ThreadPoolExecutor import yaml @@ -102,7 +105,8 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread + # Start background cleanup thread, woken by each poll tick rather than its own timer -- see OTel_Exporter/DESIGN.md. + self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() @@ -114,22 +118,23 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # `otel:` in global_controller.yaml maps straight to the OTel SDK's own - # standard env vars, not app-specific args -- the exporter subprocess itself - # stays a plain vendor-neutral OTel process; see OTel_Exporter/DESIGN.md. + # Legacy `otel:` fields map straight to the OTel SDK's own standard env vars. + # A `destinations` list is additionally passed as one Ventis-specific JSON + # variable; the exporter subprocess remains a plain OTel process otherwise. otel_env = self._otel_exporter_env(self.config.get("otel", {})) self.process_supervisor.register( "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env ) - self.process_supervisor.start_all() - # waiting table GC writes future data into (see OTel_Exporter/db.py); the - # exporter process itself calls init_db() to create the table. + # Initialize/migrate the waiting table synchronously before either the GC or + # exporter process can access it. otel_db_spec = importlib.util.spec_from_file_location( "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") ) self._otel_db = importlib.util.module_from_spec(otel_db_spec) otel_db_spec.loader.exec_module(self._otel_db) + self._otel_db.init_db() + self.process_supervisor.start_all() # ------------------------------------------------------------------ # # Stale container cleanup # @@ -177,15 +182,50 @@ def _cleanup_stale_containers(self): @staticmethod def _load_config(config_path): - """Load the YAML config file.""" + """Load the YAML config file after importing root .env values.""" + project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) + GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: return yaml.safe_load(f) + @staticmethod + def _load_dotenv(path): + """Load simple KEY=VALUE entries without overriding existing environment values.""" + if not os.path.isfile(path): + return + with open(path, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + if key and key not in os.environ: + os.environ[key] = value + + @staticmethod + def _expand_otel_value(value): + if isinstance(value, str): + return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), value) + if isinstance(value, dict): + return {key: GlobalController._expand_otel_value(item) for key, item in value.items()} + if isinstance(value, list): + return [GlobalController._expand_otel_value(item) for item in value] + return value + @staticmethod def _otel_exporter_env(otel_cfg): """Translate global_controller.yaml's `otel:` section into standard OTLP env - vars for the exporter subprocess; returns None if `otel:` is absent/empty so - the subprocess falls back to the SDK's own defaults untouched.""" + vars for the exporter subprocess. When present, ``destinations`` is passed as + JSON for the exporter to construct a fan-out. Returns None if `otel:` is + absent/empty so the subprocess falls back to the SDK's own defaults untouched. + + The legacy protocol/endpoint/headers mappings intentionally remain unchanged + for existing configurations. + """ env = {} if otel_cfg.get("protocol"): env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] @@ -195,8 +235,109 @@ def _otel_exporter_env(otel_cfg): env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( f"{k}={v}" for k, v in otel_cfg["headers"].items() ) + + if "destinations" in otel_cfg: + destinations = GlobalController._expand_otel_value(otel_cfg["destinations"]) + for destination in destinations: + if destination.get("name") == "langfuse": + public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") + secret_key = os.environ.get("LANGFUSE_SECRET_KEY") + if public_key and secret_key: + auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode() + destination.setdefault("headers", {})["Authorization"] = f"Basic {auth}" + GlobalController._validate_otel_destinations(destinations) + try: + env["VENTIS_OTEL_DESTINATIONS"] = json.dumps(destinations) + except (TypeError, ValueError) as exc: + # Do not include the offending value: destination configs commonly + # contain credentials in headers. + raise ValueError( + "otel.destinations must contain JSON-serializable values" + ) from exc return env or None + @staticmethod + def _validate_otel_destinations(destinations): + """Validate the shape of the optional exporter fan-out configuration. + + Keep this validation deliberately structural: destination-specific options + are interpreted by the exporter. Error messages identify only the location + and type, never destination contents or header values. + """ + if not isinstance(destinations, list): + raise ValueError("otel.destinations must be a list") + if not destinations: + raise ValueError("otel.destinations must not be empty") + + names = set() + for index, destination in enumerate(destinations): + if not isinstance(destination, dict): + raise ValueError( + f"otel.destinations[{index}] must be a mapping" + ) + + for field in ("name", "protocol", "endpoint"): + value = destination.get(field) + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"otel.destinations[{index}].{field} must be a non-empty string" + ) + + name = destination["name"].strip() + if name in names: + raise ValueError(f"otel.destinations contains duplicate name {name!r}") + names.add(name) + + protocol = destination["protocol"].strip().lower().replace("_", "-") + if protocol not in { + "grpc", + "otlp/grpc", + "grpc/protobuf", + "grpc-protobuf", + "http", + "http/protobuf", + "http-protobuf", + "http/proto", + "http+protobuf", + "protobuf", + }: + raise ValueError( + f"otel.destinations[{index}].protocol must be grpc or http/protobuf" + ) + + if "headers" in destination: + headers = destination["headers"] + if not isinstance(headers, dict): + raise ValueError( + f"otel.destinations[{index}].headers must be a mapping" + ) + if any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in headers.items() + ): + raise ValueError( + f"otel.destinations[{index}].headers keys and values must be strings" + ) + + if "insecure" in destination and not isinstance( + destination["insecure"], bool + ): + raise ValueError( + f"otel.destinations[{index}].insecure must be a boolean" + ) + + if "timeout" in destination: + timeout = destination["timeout"] + if ( + isinstance(timeout, bool) + or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) + or timeout <= 0 + ): + raise ValueError( + f"otel.destinations[{index}].timeout must be a positive number" + ) + @staticmethod def _get_replica_placements(ctrl): """Normalize replicas into a list of (host, port) placements.""" @@ -479,6 +620,7 @@ def run(self): self._poll_controllers() except Exception as e: logger.warning("Polling loop encountered an error: %s", e) + self._cleanup_ready.set() time.sleep(self.poll_interval) except KeyboardInterrupt: self.stop() @@ -638,9 +780,10 @@ def _get_lc_stub(self, endpoint): return self._lc_stubs[endpoint] def _cleanup_loop(self): - """Background thread: periodically trigger cleanup of completed requests.""" + """Background thread: trigger cleanup right after each poll tick, or every cleanup_interval as a fallback.""" while True: - time.sleep(self.cleanup_interval) + self._cleanup_ready.wait(timeout=self.cleanup_interval) + self._cleanup_ready.clear() try: self._trigger_cleanup() except Exception as e: From 098548c60ee297032ba7cb7c3074726955917dd3 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:24:08 -0700 Subject: [PATCH 09/27] Dedupe 'import os' from PR #51 merge (both sides added it independently) Co-Authored-By: Claude Sonnet 5 --- examples/portfolio/agents/metrics_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index d5f722e..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -10,7 +10,6 @@ # Resource profile: cheap CPU, high fan-out — one compute() call per holding. import os import sys -import os import json import math From 0b9546cbe324a99b34589e5e5b497cdad09951e4 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:49:27 -0700 Subject: [PATCH 10/27] Fix PR #51 regression: disable entrypoint-based stub relocation This project's agents/workflow import each other's stubs by flat module name, not by the exporting agent's own entrypoint path. Applying _stub_destination's entrypoint-mirroring broke both the Workflow (ModuleNotFoundError: intent_agent) and agent-to-agent calls (MetricsAgent -> price_agent) on live redeploy. Keeps PR #51's actual fix (project_dir sweep for unstubbed helper files) intact. Co-Authored-By: Claude Sonnet 5 --- ventis/cli.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 28c4fd7..920df15 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -229,8 +229,8 @@ def cmd_build(args): logger.warning("No agent YAML files found in %s", agents_dir) import yaml - - # Looks up a config entry's YAML and to map stubs to entrypoints. + + # Looks up a config entry's YAML by agent name. yaml_by_name = {} for yaml_path in yaml_files: with open(yaml_path) as f: @@ -238,13 +238,6 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path - entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} - stub_entrypoints = { - f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] - for n, p in yaml_by_name.items() - if entrypoints_by_name.get(n) - } - stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -309,7 +302,9 @@ def cmd_build(args): api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + # A workflow script imports stubs by flat module name (e.g. `from + # intent_agent import ...`), not by the agent's own entrypoint path. + stub_entrypoints=None, ) else: @@ -345,7 +340,9 @@ def cmd_build(args): stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + # Same reasoning as the workflow call above: this project's agents + # import each other's stubs by flat module name, not entrypoint path. + stub_entrypoints=None, ) bake_targets.append( From 7b3b167b27e9e8889edf099de517dc82f5662a13 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 21:40:17 -0700 Subject: [PATCH 11/27] Parallelize per-instance polling in _poll_controllers Metrics/telemetry latency scaled with instance count x per-instance round-trip time since every instance was polled sequentially, one blocking the next, with the following tick only starting after the whole pass finished. Extracted the per-instance body into _poll_one_instance (whole body wrapped in one top-level try/except, since ThreadPoolExecutor.map() re-raises on first exception when results are consumed) and run all instances concurrently via the same ThreadPoolExecutor pattern _trigger_cleanup already used. Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 16 +- ventis/controller/global_controller.py | 217 +++++++++++++------------ 2 files changed, 130 insertions(+), 103 deletions(-) diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md index 86a2a95..418ca2a 100644 --- a/OTel_Exporter/DESIGN.md +++ b/OTel_Exporter/DESIGN.md @@ -167,7 +167,21 @@ Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall the poll loop's health checks and OTel writes. -### 6. Dependencies (all added) +### 6. Parallelized per-instance polling (`ventis/controller/global_controller.py`) +`_poll_controllers` used to loop over every instance sequentially -- Redis reads, an +OTel sqlite write, and up to two Postgres writes per instance, one instance fully +blocking the next, with the following poll tick only starting after the whole pass +finished. Total metrics/telemetry latency scaled with instance count x round-trip +time, not the configured `poll_interval`. Fixed by extracting the per-instance body +into `_poll_one_instance` (its whole body wrapped in one top-level try/except, since +`ThreadPoolExecutor.map()` re-raises on first exception when results are consumed) +and running all instances concurrently via the same `ThreadPoolExecutor` pattern +`_trigger_cleanup` already used. Known, pre-existing, previously acknowledged in +commit `a6694d9`'s own message but never actually fixed (a same-named follow-up +branch was found to contain no real threading changes) -- see company-memory for +the investigation. + +### 7. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index bdec683..da29783 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -643,118 +643,131 @@ def _poll_controllers(self): if self.running: self.process_supervisor.check_and_respawn() - for instance in self.instance_manager.list_instances(): + # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer + # gates every other instance's poll -- see OTel_Exporter/DESIGN.md. + instances = self.instance_manager.list_instances() + if instances: + with ThreadPoolExecutor(max_workers=len(instances)) as executor: + list(executor.map(self._poll_one_instance, instances)) + + def _poll_one_instance(self, instance): + """Poll and persist one instance's runtime/metrics/health data; never raises.""" + try: name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, + except Exception as e: + logger.warning("Failed to poll instance %s: %s", instance, e) + return + + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" + + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now - - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, - ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) - else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status + else: + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From b5d6e4dcbc6aee7265e1dc01080f70da39eb90f2 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 28 Aug 2026 13:58:05 -0700 Subject: [PATCH 12/27] rough draft --- .../portfolio/config/global_controller.yaml | 25 +++++---- .../portfolio/workflow/portfolio_workflow.py | 1 - pyproject.toml | 2 +- requirements.txt | 2 - tests/test_otel_exporter_fanout.py | 2 +- tests/test_otel_exporter_fields.py | 2 +- .../OTLP_Exporter}/DESIGN.md | 6 +- ventis/OTLP_Exporter/SCHEMA.md | 56 +++++++++++++++++++ .../OTLP_Exporter}/__init__.py | 0 .../OTLP_Exporter}/convert.py | 24 ++++++-- {OTel_Exporter => ventis/OTLP_Exporter}/db.py | 4 +- .../OTLP_Exporter}/otel_exporter.py | 0 ventis/OTLP_Exporter/otel_queue.db | 0 ventis/cli.py | 21 +++++-- .../cloud_provider_logic/EC2/_runtime.py | 5 +- ventis/controller/global_controller.py | 27 +++++---- ventis/controller/utils/process_supervisor.py | 2 +- ventis/stub_generator.py | 40 +++++++++++-- 18 files changed, 167 insertions(+), 52 deletions(-) rename {OTel_Exporter => ventis/OTLP_Exporter}/DESIGN.md (98%) create mode 100644 ventis/OTLP_Exporter/SCHEMA.md rename {OTel_Exporter => ventis/OTLP_Exporter}/__init__.py (100%) rename {OTel_Exporter => ventis/OTLP_Exporter}/convert.py (72%) rename {OTel_Exporter => ventis/OTLP_Exporter}/db.py (98%) rename {OTel_Exporter => ventis/OTLP_Exporter}/otel_exporter.py (100%) create mode 100644 ventis/OTLP_Exporter/otel_queue.db diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index ab5af4d..edee85f 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -83,18 +83,18 @@ otel: destinations: - name: railway protocol: grpc - endpoint: yamanote.proxy.rlwy.net:19803 + endpoint: ${RAILWAY_OTLP_ENDPOINT} insecure: true headers: {} - - name: langfuse - protocol: http - endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces - headers: {} - name: grafana protocol: http endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} +# - name: langfuse +# protocol: http +# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces +# headers: {} # Polling interval in seconds poll_interval: 5 @@ -107,10 +107,13 @@ redis: # EC2 defaults for `provider: EC2` replicas. ec2: - region: us-east-1 - ami_id: ami-031ff6df47f26b546 - subnet_id: subnet-0638ac6d79d488124 + region: ${EC2_REGION} + ami_id: ${EC2_AMI_ID} + subnet_id: ${EC2_SUBNET_ID} security_group_ids: - - sg-025daf3a98e06cef3 - ssh_user: ubuntu - ssh_private_key_path: ~/.ssh/ventis_ec2 + - ${EC2_SECURITY_GROUP_ID} + ssh_user: ${EC2_SSH_USER} + ssh_private_key_path: ${EC2_SSH_PRIVATE_KEY_PATH} + +database: + url: ${DATABASE_URL} diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 299a3f5..b8b684a 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -46,7 +46,6 @@ def main( advisor = AdvisorAgent() # Stage 0: parse the free-text request into structured holdings + window. - intent = intent_agent.parse(query=query).value() # parse() returns a dict, but a Future's .value() only ever gives back the # raw string ventis stored in Redis -- same deserialization requirement as # every other dict-returning agent call below. diff --git a/pyproject.toml b/pyproject.toml index 9ae1234..40cb43a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["ventis*", "OTel_Exporter*"] +include = ["ventis*"] [tool.setuptools.package-data] ventis = [ diff --git a/requirements.txt b/requirements.txt index dd7a254..f06e0b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,6 @@ grpcio-tools redis pyyaml flask -ipdb -ipython sqlalchemy psycopg[binary] psutil diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index ed1b423..0691d61 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -13,7 +13,7 @@ ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # ``otel_exporter.py`` is also executed as a script from its own directory and # therefore imports ``convert`` and ``db`` as top-level modules. -sys.path.insert(0, os.path.join(ROOT, "OTel_Exporter")) +sys.path.insert(0, os.path.join(ROOT, "ventis", "OTLP_Exporter")) import db # noqa: E402 import otel_exporter # noqa: E402 diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index 4a62c58..ff8cad6 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -5,7 +5,7 @@ import unittest from unittest.mock import patch -from OTel_Exporter import convert, db +from ventis.OTLP_Exporter import convert, db class OTelExporterFieldTests(unittest.TestCase): diff --git a/OTel_Exporter/DESIGN.md b/ventis/OTLP_Exporter/DESIGN.md similarity index 98% rename from OTel_Exporter/DESIGN.md rename to ventis/OTLP_Exporter/DESIGN.md index 418ca2a..d433fe0 100644 --- a/OTel_Exporter/DESIGN.md +++ b/ventis/OTLP_Exporter/DESIGN.md @@ -48,7 +48,7 @@ Decisions (final status): Configuration is read at exporter startup; changing it requires a GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own - SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing + SQLite file (`ventis/OTLP_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing `send_runtime_information` write. Keeps this pipeline's schema/state fully decoupled from the dashboard/cost table. @@ -95,7 +95,7 @@ endpoint and headers to the SDK. `BatchSpanProcessor(..., schedule_delay_millis= `max_export_batch_size` is left at the SDK default (512), which already approximates the original "500 spans" batching ask without any override needed. -### 2. `OTel_Exporter/otel_exporter.py` +### 2. `ventis/OTLP_Exporter/otel_exporter.py` A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM stays responsive), calling `_send_pending()` each tick. At startup it constructs one independent OTLP exporter and `BatchSpanProcessor` for each configured destination; @@ -111,7 +111,7 @@ each pair may use a different protocol, endpoint, and headers: since spans are hand-built and handed straight to the processors via `on_end()`. - Every processor is shut down on exit, flushing its pending batch independently. -### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) +### 3. Future row → OTel span conversion (`ventis/OTLP_Exporter/convert.py`) `future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is the one that maps to `trace_id`. Both are `uuid4().hex` (32 hex chars / 16 bytes); OTel `trace_id` is 128-bit (16 bytes, fits directly) and `span_id` is 64-bit (8 bytes, needs diff --git a/ventis/OTLP_Exporter/SCHEMA.md b/ventis/OTLP_Exporter/SCHEMA.md new file mode 100644 index 0000000..38f8664 --- /dev/null +++ b/ventis/OTLP_Exporter/SCHEMA.md @@ -0,0 +1,56 @@ +# OTel span storage schema + +Each emitted OTel span is stored as one `otel_spans` record. This chart also defines a +one-to-one `otel_span_attributes` projection, linked by the same `span_id`, for the +known exporter attributes. The current receiver retains the raw `attributes` JSONB map; +the child table is the relational schema described in `otel_spans_schema.txt`. + +```text +┌───────────────────────────┐ ┌────────────────────────────────┐ +│ otel_spans │ │ otel_span_attributes │ +├───────────────────────────┤ ├────────────────────────────────┤ +│ PK span_id │────1:1───│ PK/FK span_id │ +│ trace_id │ │ model and agent ID │ +│ parent_span_id │ │ CPU and GPU │ +│ name │ │ timing and token usage │ +│ kind │ │ input and output │ +│ start/end time (ns) │ │ project and error count │ +│ status code/message │ │ server/token/total cost │ +│ attributes (JSONB) │ │ cache tokens/hit ratio │ +│ events (JSONB) │ └────────────────────────────────┘ +└───────────────────────────┘ +``` + +## `otel_spans` + +| Column | Meaning | +| --- | --- | +| `span_id` | Unique identifier for this span. | +| `trace_id` | Identifier shared by all spans in the same trace. | +| `parent_span_id` | Parent span; empty for a root span. | +| `name` | Operation name, such as an agent method. | +| `kind` | OTel role of the work; current exporter spans are `SPAN_KIND_INTERNAL`. | +| `start_time_unix_nano` / `end_time_unix_nano` | Raw Unix timestamps in nanoseconds. | +| `status_code` / `status_message` | OTel outcome: normally `STATUS_CODE_UNSET`, or `STATUS_CODE_ERROR` with an error message. | +| `attributes` | Complete raw OTel attribute map (JSONB). | +| `events` | OTel events, including any `exception` event (JSONB). | + +## `otel_span_attributes` + +This one-to-one projection mirrors every attribute currently emitted by +`convert.waiting_row_to_span()`. Fields are nullable because OTel omits an attribute +whose source value is `None`. + +| Group | Columns | +| --- | --- | +| Model and agent | `gen_ai.request.model`, `gen_ai.agent.id` | +| Resources and timing | `cpu`, `gpu`, `execution_time_ms`, `queue_time_ms` | +| Token usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `token_count`, `gen_ai.usage.cache_read.input_tokens` | +| Input and output | `langfuse.observation.input`, `langfuse.observation.output` | +| Project and errors | `project_id`, `error_count` | +| Costs | `server_cost`, `token_cost`, `gen_ai.usage.cost` | +| Cache | `cache_hit_ratio` | + +The DBML source is [`../otel_spans_schema.txt`](../otel_spans_schema.txt). + +Successful spans use `STATUS_CODE_UNSET` rather than `STATUS_CODE_OK` because OpenTelemetry reserves `OK` for application- or operator-validated success, while instrumentation normally sets a status only when it records an error. diff --git a/OTel_Exporter/__init__.py b/ventis/OTLP_Exporter/__init__.py similarity index 100% rename from OTel_Exporter/__init__.py rename to ventis/OTLP_Exporter/__init__.py diff --git a/OTel_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py similarity index 72% rename from OTel_Exporter/convert.py rename to ventis/OTLP_Exporter/convert.py index b7fb493..66da972 100644 --- a/OTel_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -3,7 +3,7 @@ Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects directly instead of going through Tracer.start_span() -- there's no live tracer here, futures already finished (sometimes in another process), so this is a historical-row -conversion, not live tracing. See OTel_Exporter/DESIGN.md for why this deviates from +conversion, not live tracing. See ventis/OTLP_Exporter/DESIGN.md for why this deviates from the SDK's usual advice against constructing ReadableSpan by hand. """ @@ -64,9 +64,17 @@ def waiting_row_to_span(row): ) status = Status(StatusCode.ERROR, description=row.get("error_message")) - # Model and token usage use OTel GenAI semantic-convention names. Observation - # input/output use Langfuse's documented JSON-string attributes. The remaining - # Ventis infrastructure values have no GenAI equivalent, so they keep plain names. + # Model, token, agent, and cache-read usage use OTel GenAI semantic-convention + # names (see https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/). + # total_cost uses gen_ai.usage.cost, which isn't yet an official semconv attribute + # but is the name Langfuse's OTel ingestion actually reads (its JSON cost_details + # attribute is currently broken -- see langfuse/langfuse#11030). Observation + # input/output use Langfuse's documented JSON-string attributes. `errors` is named + # error_count, not "errors"/"error", to avoid colliding with OTel's reserved + # error.* namespace (error.type etc.), which describes a single error, not a + # count. The remaining Ventis-specific values (project_id, server/token cost + # breakdown, cache_hit_ratio) have no GenAI or Langfuse equivalent, so they keep + # plain names. attributes = { k: v for k, v in { @@ -80,6 +88,14 @@ def waiting_row_to_span(row): "token_count": row.get("token_count"), "langfuse.observation.input": row.get("input"), "langfuse.observation.output": row.get("output"), + "project_id": row.get("project_id"), + "gen_ai.agent.id": row.get("agent_id"), + "error_count": row.get("errors"), + "server_cost": row.get("server_cost"), + "token_cost": row.get("token_cost"), + "gen_ai.usage.cost": row.get("total_cost"), + "gen_ai.usage.cache_read.input_tokens": row.get("cached_tokens"), + "cache_hit_ratio": row.get("cache_hit_ratio"), }.items() if v is not None } diff --git a/OTel_Exporter/db.py b/ventis/OTLP_Exporter/db.py similarity index 98% rename from OTel_Exporter/db.py rename to ventis/OTLP_Exporter/db.py index a8a675f..e6a1834 100644 --- a/OTel_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -130,7 +130,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH if not fid or not session_id: continue agent_id = raw.get("agent") - started_at = float(raw.get("created_at") or 0) or None + started_at = float(raw.get("created_at") or 0) finished_at = float(raw["finished_at"]) if raw.get("finished_at") else None execution_time_ms = ( round((finished_at - started_at) * 1000) @@ -160,7 +160,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH * _TOKEN_COST_MULTIPLIER ) # Server cost needs an elapsed duration -- only available once finished. - if finished_at and started_at: + if finished_at is not None: server_cost = ( pricing.compute_server_cost( redis_client.get(f"agent:{agent_id}:instance_type") diff --git a/OTel_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py similarity index 100% rename from OTel_Exporter/otel_exporter.py rename to ventis/OTLP_Exporter/otel_exporter.py diff --git a/ventis/OTLP_Exporter/otel_queue.db b/ventis/OTLP_Exporter/otel_queue.db new file mode 100644 index 0000000..e69de29 diff --git a/ventis/cli.py b/ventis/cli.py index 920df15..31a32e0 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -238,6 +238,15 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path + # Maps each generated stub's basename to its agent's entrypoint path, so a + # stub can also be placed at its nested, entrypoint-mirrored location. + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -302,9 +311,9 @@ def cmd_build(args): api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # A workflow script imports stubs by flat module name (e.g. `from - # intent_agent import ...`), not by the agent's own entrypoint path. - stub_entrypoints=None, + # Stubs are placed both flat and at their entrypoint-mirrored path, + # so both flat and nested import styles resolve to the stub. + stub_entrypoints=stub_entrypoints, ) else: @@ -340,9 +349,9 @@ def cmd_build(args): stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # Same reasoning as the workflow call above: this project's agents - # import each other's stubs by flat module name, not entrypoint path. - stub_entrypoints=None, + # Same reasoning as the workflow call above: stubs are placed both + # flat and at their entrypoint-mirrored path. + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 9955fa2..7e4e9ab 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -239,11 +239,12 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, logger.info("Transferring image %s to %s", image, host) result = subprocess.run( "set -o pipefail; " - f"docker save {shlex.quote(image)} | ssh -o StrictHostKeyChecking=no " + f"docker save {shlex.quote(image)} | zstd -T0 | ssh -o StrictHostKeyChecking=no " f"-o IdentitiesOnly=yes -o ConnectTimeout=10 " f"-o ServerAliveInterval=10 -o ServerAliveCountMax=3 " f"-i {shlex.quote(key)} " - f"{shlex.quote(f'{ssh_user}@{host}')} 'sudo docker load'", + f"{shlex.quote(f'{ssh_user}@{host}')} " + "'set -o pipefail; zstd -d | sudo docker load'", shell=True, capture_output=True, text=True, diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index da29783..6584ef8 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -110,16 +110,16 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread, woken by each poll tick rather than its own timer -- see OTel_Exporter/DESIGN.md. + # Start background cleanup thread, woken by each poll tick rather than its own timer -- see ventis/OTLP_Exporter/DESIGN.md. self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() - # Spawn the OTLP exporter as a separate process (see OTel_Exporter/DESIGN.md), + # Spawn the OTLP exporter as a separate process (see ventis/OTLP_Exporter/DESIGN.md), # supervised so it gets restarted if it ever exits unexpectedly. otel_exporter_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "OTel_Exporter", + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "OTLP_Exporter", ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() @@ -191,7 +191,12 @@ def _load_config(config_path): project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: - return yaml.safe_load(f) + config = yaml.safe_load(f) + if "ec2" in config: + config["ec2"] = GlobalController._expand_env_value(config["ec2"]) + if "database" in config: + config["database"] = GlobalController._expand_env_value(config["database"]) + return config @staticmethod def _load_dotenv(path): @@ -212,13 +217,13 @@ def _load_dotenv(path): os.environ[key] = value @staticmethod - def _expand_otel_value(value): + def _expand_env_value(value): if isinstance(value, str): return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), value) if isinstance(value, dict): - return {key: GlobalController._expand_otel_value(item) for key, item in value.items()} + return {key: GlobalController._expand_env_value(item) for key, item in value.items()} if isinstance(value, list): - return [GlobalController._expand_otel_value(item) for item in value] + return [GlobalController._expand_env_value(item) for item in value] return value @staticmethod @@ -242,7 +247,7 @@ def _otel_exporter_env(otel_cfg): ) if "destinations" in otel_cfg: - destinations = GlobalController._expand_otel_value(otel_cfg["destinations"]) + destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) for destination in destinations: if destination.get("name") == "langfuse": public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") @@ -639,12 +644,12 @@ def _poll_controllers(self): # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which # terminates every managed process) via the signal handler before this line is # reached -- without the guard, this could respawn a process just intentionally - # killed. See OTel_Exporter/DESIGN.md. + # killed. See ventis/OTLP_Exporter/DESIGN.md. if self.running: self.process_supervisor.check_and_respawn() # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer - # gates every other instance's poll -- see OTel_Exporter/DESIGN.md. + # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. instances = self.instance_manager.list_instances() if instances: with ThreadPoolExecutor(max_workers=len(instances)) as executor: diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 8f5724c..8c061bc 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -3,7 +3,7 @@ register() + start_all() spawn processes; check_and_respawn() (call from GC's existing poll tick) restarts any that exit unexpectedly; terminate_all() (call from GC's shutdown path) stops them all cleanly. Deliberately GC-agnostic -- callers are responsible for not -calling check_and_respawn() during their own shutdown (see OTel_Exporter/DESIGN.md's +calling check_and_respawn() during their own shutdown (see ventis/OTLP_Exporter/DESIGN.md's shutdown-race note). """ diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 803fc2d..4647619 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -17,9 +17,7 @@ import yaml # Packages every agent container needs regardless of its specific business logic. -# grpcio-tools/pyyaml/ipdb/ipython aren't needed/used, but keeping to keep the scope constrained right now -# - Leave a comment if you want me to remove these, I kept them in since you originally had them but they aren't used -BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "ipdb", "ipython", "boto3"] +BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "boto3"] # Workflow will always require these BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask", "sqlalchemy", "psycopg[binary]"] @@ -322,6 +320,21 @@ def _copy_files(output_dir, files_to_copy): shutil.copy2(src, dest_path) +def _write_entrypoint_file(src, dest_path, project_dir): + """Copy an entrypoint file to dest_path, injecting a sys.path entry for its + original sibling directory so a co-located, non-stub helper import still resolves.""" + original_dir = os.path.dirname(os.path.relpath(src, project_dir)) if project_dir else "" + if not original_dir: + shutil.copy2(src, dest_path) + return + injection = ( + f"import sys, os\n" + f"sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), {original_dir!r}))\n" + ) + with open(src) as f, open(dest_path, "w") as out: + out.write(injection + f.read()) + + def generate_docker( yaml_path, agent_file, @@ -405,8 +418,7 @@ def generate_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - - files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -416,6 +428,14 @@ def generate_docker( _copy_files(output_dir, files_to_copy) + # Copy the agent's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(agent_file), + os.path.join(output_dir, os.path.basename(agent_file)), + project_dir, + ) + # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -499,7 +519,6 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ - (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), (os.path.join(script_dir, "deploy.py"), "deploy.py"), @@ -527,6 +546,7 @@ def generate_workflow_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -536,6 +556,14 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) + # Copy the workflow's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(workflow_file), + os.path.join(output_dir, workflow_basename), + project_dir, + ) + # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time From 9bbc35d2206016e72b65061579d7bf2e62098c47 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 28 Aug 2026 13:58:05 -0700 Subject: [PATCH 13/27] rough draft --- .../portfolio/config/global_controller.yaml | 25 +++++---- .../portfolio/workflow/portfolio_workflow.py | 1 - pyproject.toml | 2 +- requirements.txt | 2 - tests/test_otel_exporter_fanout.py | 2 +- tests/test_otel_exporter_fields.py | 2 +- .../OTLP_Exporter}/DESIGN.md | 6 +- ventis/OTLP_Exporter/SCHEMA.md | 56 +++++++++++++++++++ .../OTLP_Exporter}/__init__.py | 0 .../OTLP_Exporter}/convert.py | 24 ++++++-- {OTel_Exporter => ventis/OTLP_Exporter}/db.py | 4 +- .../OTLP_Exporter}/otel_exporter.py | 0 ventis/OTLP_Exporter/otel_queue.db | 0 ventis/cli.py | 21 +++++-- .../cloud_provider_logic/EC2/_runtime.py | 5 +- ventis/controller/global_controller.py | 25 +++++---- ventis/controller/utils/process_supervisor.py | 2 +- ventis/stub_generator.py | 40 +++++++++++-- 18 files changed, 166 insertions(+), 51 deletions(-) rename {OTel_Exporter => ventis/OTLP_Exporter}/DESIGN.md (98%) create mode 100644 ventis/OTLP_Exporter/SCHEMA.md rename {OTel_Exporter => ventis/OTLP_Exporter}/__init__.py (100%) rename {OTel_Exporter => ventis/OTLP_Exporter}/convert.py (72%) rename {OTel_Exporter => ventis/OTLP_Exporter}/db.py (98%) rename {OTel_Exporter => ventis/OTLP_Exporter}/otel_exporter.py (100%) create mode 100644 ventis/OTLP_Exporter/otel_queue.db diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index ab5af4d..edee85f 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -83,18 +83,18 @@ otel: destinations: - name: railway protocol: grpc - endpoint: yamanote.proxy.rlwy.net:19803 + endpoint: ${RAILWAY_OTLP_ENDPOINT} insecure: true headers: {} - - name: langfuse - protocol: http - endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces - headers: {} - name: grafana protocol: http endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} +# - name: langfuse +# protocol: http +# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces +# headers: {} # Polling interval in seconds poll_interval: 5 @@ -107,10 +107,13 @@ redis: # EC2 defaults for `provider: EC2` replicas. ec2: - region: us-east-1 - ami_id: ami-031ff6df47f26b546 - subnet_id: subnet-0638ac6d79d488124 + region: ${EC2_REGION} + ami_id: ${EC2_AMI_ID} + subnet_id: ${EC2_SUBNET_ID} security_group_ids: - - sg-025daf3a98e06cef3 - ssh_user: ubuntu - ssh_private_key_path: ~/.ssh/ventis_ec2 + - ${EC2_SECURITY_GROUP_ID} + ssh_user: ${EC2_SSH_USER} + ssh_private_key_path: ${EC2_SSH_PRIVATE_KEY_PATH} + +database: + url: ${DATABASE_URL} diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 299a3f5..b8b684a 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -46,7 +46,6 @@ def main( advisor = AdvisorAgent() # Stage 0: parse the free-text request into structured holdings + window. - intent = intent_agent.parse(query=query).value() # parse() returns a dict, but a Future's .value() only ever gives back the # raw string ventis stored in Redis -- same deserialization requirement as # every other dict-returning agent call below. diff --git a/pyproject.toml b/pyproject.toml index 9ae1234..40cb43a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["ventis*", "OTel_Exporter*"] +include = ["ventis*"] [tool.setuptools.package-data] ventis = [ diff --git a/requirements.txt b/requirements.txt index dd7a254..f06e0b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,6 @@ grpcio-tools redis pyyaml flask -ipdb -ipython sqlalchemy psycopg[binary] psutil diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index ed1b423..0691d61 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -13,7 +13,7 @@ ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # ``otel_exporter.py`` is also executed as a script from its own directory and # therefore imports ``convert`` and ``db`` as top-level modules. -sys.path.insert(0, os.path.join(ROOT, "OTel_Exporter")) +sys.path.insert(0, os.path.join(ROOT, "ventis", "OTLP_Exporter")) import db # noqa: E402 import otel_exporter # noqa: E402 diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index 4a62c58..ff8cad6 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -5,7 +5,7 @@ import unittest from unittest.mock import patch -from OTel_Exporter import convert, db +from ventis.OTLP_Exporter import convert, db class OTelExporterFieldTests(unittest.TestCase): diff --git a/OTel_Exporter/DESIGN.md b/ventis/OTLP_Exporter/DESIGN.md similarity index 98% rename from OTel_Exporter/DESIGN.md rename to ventis/OTLP_Exporter/DESIGN.md index 86a2a95..287fffd 100644 --- a/OTel_Exporter/DESIGN.md +++ b/ventis/OTLP_Exporter/DESIGN.md @@ -48,7 +48,7 @@ Decisions (final status): Configuration is read at exporter startup; changing it requires a GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own - SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing + SQLite file (`ventis/OTLP_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing `send_runtime_information` write. Keeps this pipeline's schema/state fully decoupled from the dashboard/cost table. @@ -95,7 +95,7 @@ endpoint and headers to the SDK. `BatchSpanProcessor(..., schedule_delay_millis= `max_export_batch_size` is left at the SDK default (512), which already approximates the original "500 spans" batching ask without any override needed. -### 2. `OTel_Exporter/otel_exporter.py` +### 2. `ventis/OTLP_Exporter/otel_exporter.py` A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM stays responsive), calling `_send_pending()` each tick. At startup it constructs one independent OTLP exporter and `BatchSpanProcessor` for each configured destination; @@ -111,7 +111,7 @@ each pair may use a different protocol, endpoint, and headers: since spans are hand-built and handed straight to the processors via `on_end()`. - Every processor is shut down on exit, flushing its pending batch independently. -### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) +### 3. Future row → OTel span conversion (`ventis/OTLP_Exporter/convert.py`) `future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is the one that maps to `trace_id`. Both are `uuid4().hex` (32 hex chars / 16 bytes); OTel `trace_id` is 128-bit (16 bytes, fits directly) and `span_id` is 64-bit (8 bytes, needs diff --git a/ventis/OTLP_Exporter/SCHEMA.md b/ventis/OTLP_Exporter/SCHEMA.md new file mode 100644 index 0000000..38f8664 --- /dev/null +++ b/ventis/OTLP_Exporter/SCHEMA.md @@ -0,0 +1,56 @@ +# OTel span storage schema + +Each emitted OTel span is stored as one `otel_spans` record. This chart also defines a +one-to-one `otel_span_attributes` projection, linked by the same `span_id`, for the +known exporter attributes. The current receiver retains the raw `attributes` JSONB map; +the child table is the relational schema described in `otel_spans_schema.txt`. + +```text +┌───────────────────────────┐ ┌────────────────────────────────┐ +│ otel_spans │ │ otel_span_attributes │ +├───────────────────────────┤ ├────────────────────────────────┤ +│ PK span_id │────1:1───│ PK/FK span_id │ +│ trace_id │ │ model and agent ID │ +│ parent_span_id │ │ CPU and GPU │ +│ name │ │ timing and token usage │ +│ kind │ │ input and output │ +│ start/end time (ns) │ │ project and error count │ +│ status code/message │ │ server/token/total cost │ +│ attributes (JSONB) │ │ cache tokens/hit ratio │ +│ events (JSONB) │ └────────────────────────────────┘ +└───────────────────────────┘ +``` + +## `otel_spans` + +| Column | Meaning | +| --- | --- | +| `span_id` | Unique identifier for this span. | +| `trace_id` | Identifier shared by all spans in the same trace. | +| `parent_span_id` | Parent span; empty for a root span. | +| `name` | Operation name, such as an agent method. | +| `kind` | OTel role of the work; current exporter spans are `SPAN_KIND_INTERNAL`. | +| `start_time_unix_nano` / `end_time_unix_nano` | Raw Unix timestamps in nanoseconds. | +| `status_code` / `status_message` | OTel outcome: normally `STATUS_CODE_UNSET`, or `STATUS_CODE_ERROR` with an error message. | +| `attributes` | Complete raw OTel attribute map (JSONB). | +| `events` | OTel events, including any `exception` event (JSONB). | + +## `otel_span_attributes` + +This one-to-one projection mirrors every attribute currently emitted by +`convert.waiting_row_to_span()`. Fields are nullable because OTel omits an attribute +whose source value is `None`. + +| Group | Columns | +| --- | --- | +| Model and agent | `gen_ai.request.model`, `gen_ai.agent.id` | +| Resources and timing | `cpu`, `gpu`, `execution_time_ms`, `queue_time_ms` | +| Token usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `token_count`, `gen_ai.usage.cache_read.input_tokens` | +| Input and output | `langfuse.observation.input`, `langfuse.observation.output` | +| Project and errors | `project_id`, `error_count` | +| Costs | `server_cost`, `token_cost`, `gen_ai.usage.cost` | +| Cache | `cache_hit_ratio` | + +The DBML source is [`../otel_spans_schema.txt`](../otel_spans_schema.txt). + +Successful spans use `STATUS_CODE_UNSET` rather than `STATUS_CODE_OK` because OpenTelemetry reserves `OK` for application- or operator-validated success, while instrumentation normally sets a status only when it records an error. diff --git a/OTel_Exporter/__init__.py b/ventis/OTLP_Exporter/__init__.py similarity index 100% rename from OTel_Exporter/__init__.py rename to ventis/OTLP_Exporter/__init__.py diff --git a/OTel_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py similarity index 72% rename from OTel_Exporter/convert.py rename to ventis/OTLP_Exporter/convert.py index b7fb493..66da972 100644 --- a/OTel_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -3,7 +3,7 @@ Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects directly instead of going through Tracer.start_span() -- there's no live tracer here, futures already finished (sometimes in another process), so this is a historical-row -conversion, not live tracing. See OTel_Exporter/DESIGN.md for why this deviates from +conversion, not live tracing. See ventis/OTLP_Exporter/DESIGN.md for why this deviates from the SDK's usual advice against constructing ReadableSpan by hand. """ @@ -64,9 +64,17 @@ def waiting_row_to_span(row): ) status = Status(StatusCode.ERROR, description=row.get("error_message")) - # Model and token usage use OTel GenAI semantic-convention names. Observation - # input/output use Langfuse's documented JSON-string attributes. The remaining - # Ventis infrastructure values have no GenAI equivalent, so they keep plain names. + # Model, token, agent, and cache-read usage use OTel GenAI semantic-convention + # names (see https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/). + # total_cost uses gen_ai.usage.cost, which isn't yet an official semconv attribute + # but is the name Langfuse's OTel ingestion actually reads (its JSON cost_details + # attribute is currently broken -- see langfuse/langfuse#11030). Observation + # input/output use Langfuse's documented JSON-string attributes. `errors` is named + # error_count, not "errors"/"error", to avoid colliding with OTel's reserved + # error.* namespace (error.type etc.), which describes a single error, not a + # count. The remaining Ventis-specific values (project_id, server/token cost + # breakdown, cache_hit_ratio) have no GenAI or Langfuse equivalent, so they keep + # plain names. attributes = { k: v for k, v in { @@ -80,6 +88,14 @@ def waiting_row_to_span(row): "token_count": row.get("token_count"), "langfuse.observation.input": row.get("input"), "langfuse.observation.output": row.get("output"), + "project_id": row.get("project_id"), + "gen_ai.agent.id": row.get("agent_id"), + "error_count": row.get("errors"), + "server_cost": row.get("server_cost"), + "token_cost": row.get("token_cost"), + "gen_ai.usage.cost": row.get("total_cost"), + "gen_ai.usage.cache_read.input_tokens": row.get("cached_tokens"), + "cache_hit_ratio": row.get("cache_hit_ratio"), }.items() if v is not None } diff --git a/OTel_Exporter/db.py b/ventis/OTLP_Exporter/db.py similarity index 98% rename from OTel_Exporter/db.py rename to ventis/OTLP_Exporter/db.py index a8a675f..e6a1834 100644 --- a/OTel_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -130,7 +130,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH if not fid or not session_id: continue agent_id = raw.get("agent") - started_at = float(raw.get("created_at") or 0) or None + started_at = float(raw.get("created_at") or 0) finished_at = float(raw["finished_at"]) if raw.get("finished_at") else None execution_time_ms = ( round((finished_at - started_at) * 1000) @@ -160,7 +160,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH * _TOKEN_COST_MULTIPLIER ) # Server cost needs an elapsed duration -- only available once finished. - if finished_at and started_at: + if finished_at is not None: server_cost = ( pricing.compute_server_cost( redis_client.get(f"agent:{agent_id}:instance_type") diff --git a/OTel_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py similarity index 100% rename from OTel_Exporter/otel_exporter.py rename to ventis/OTLP_Exporter/otel_exporter.py diff --git a/ventis/OTLP_Exporter/otel_queue.db b/ventis/OTLP_Exporter/otel_queue.db new file mode 100644 index 0000000..e69de29 diff --git a/ventis/cli.py b/ventis/cli.py index 920df15..31a32e0 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -238,6 +238,15 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path + # Maps each generated stub's basename to its agent's entrypoint path, so a + # stub can also be placed at its nested, entrypoint-mirrored location. + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -302,9 +311,9 @@ def cmd_build(args): api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # A workflow script imports stubs by flat module name (e.g. `from - # intent_agent import ...`), not by the agent's own entrypoint path. - stub_entrypoints=None, + # Stubs are placed both flat and at their entrypoint-mirrored path, + # so both flat and nested import styles resolve to the stub. + stub_entrypoints=stub_entrypoints, ) else: @@ -340,9 +349,9 @@ def cmd_build(args): stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # Same reasoning as the workflow call above: this project's agents - # import each other's stubs by flat module name, not entrypoint path. - stub_entrypoints=None, + # Same reasoning as the workflow call above: stubs are placed both + # flat and at their entrypoint-mirrored path. + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 9955fa2..7e4e9ab 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -239,11 +239,12 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, logger.info("Transferring image %s to %s", image, host) result = subprocess.run( "set -o pipefail; " - f"docker save {shlex.quote(image)} | ssh -o StrictHostKeyChecking=no " + f"docker save {shlex.quote(image)} | zstd -T0 | ssh -o StrictHostKeyChecking=no " f"-o IdentitiesOnly=yes -o ConnectTimeout=10 " f"-o ServerAliveInterval=10 -o ServerAliveCountMax=3 " f"-i {shlex.quote(key)} " - f"{shlex.quote(f'{ssh_user}@{host}')} 'sudo docker load'", + f"{shlex.quote(f'{ssh_user}@{host}')} " + "'set -o pipefail; zstd -d | sudo docker load'", shell=True, capture_output=True, text=True, diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index bdec683..16e0e9c 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -110,16 +110,16 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread, woken by each poll tick rather than its own timer -- see OTel_Exporter/DESIGN.md. + # Start background cleanup thread, woken by each poll tick rather than its own timer -- see ventis/OTLP_Exporter/DESIGN.md. self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() - # Spawn the OTLP exporter as a separate process (see OTel_Exporter/DESIGN.md), + # Spawn the OTLP exporter as a separate process (see ventis/OTLP_Exporter/DESIGN.md), # supervised so it gets restarted if it ever exits unexpectedly. otel_exporter_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "OTel_Exporter", + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "OTLP_Exporter", ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() @@ -191,7 +191,12 @@ def _load_config(config_path): project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: - return yaml.safe_load(f) + config = yaml.safe_load(f) + if "ec2" in config: + config["ec2"] = GlobalController._expand_env_value(config["ec2"]) + if "database" in config: + config["database"] = GlobalController._expand_env_value(config["database"]) + return config @staticmethod def _load_dotenv(path): @@ -212,13 +217,13 @@ def _load_dotenv(path): os.environ[key] = value @staticmethod - def _expand_otel_value(value): + def _expand_env_value(value): if isinstance(value, str): return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), value) if isinstance(value, dict): - return {key: GlobalController._expand_otel_value(item) for key, item in value.items()} + return {key: GlobalController._expand_env_value(item) for key, item in value.items()} if isinstance(value, list): - return [GlobalController._expand_otel_value(item) for item in value] + return [GlobalController._expand_env_value(item) for item in value] return value @staticmethod @@ -242,7 +247,7 @@ def _otel_exporter_env(otel_cfg): ) if "destinations" in otel_cfg: - destinations = GlobalController._expand_otel_value(otel_cfg["destinations"]) + destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) for destination in destinations: if destination.get("name") == "langfuse": public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") @@ -639,7 +644,7 @@ def _poll_controllers(self): # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which # terminates every managed process) via the signal handler before this line is # reached -- without the guard, this could respawn a process just intentionally - # killed. See OTel_Exporter/DESIGN.md. + # killed. See ventis/OTLP_Exporter/DESIGN.md. if self.running: self.process_supervisor.check_and_respawn() diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 8f5724c..8c061bc 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -3,7 +3,7 @@ register() + start_all() spawn processes; check_and_respawn() (call from GC's existing poll tick) restarts any that exit unexpectedly; terminate_all() (call from GC's shutdown path) stops them all cleanly. Deliberately GC-agnostic -- callers are responsible for not -calling check_and_respawn() during their own shutdown (see OTel_Exporter/DESIGN.md's +calling check_and_respawn() during their own shutdown (see ventis/OTLP_Exporter/DESIGN.md's shutdown-race note). """ diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 803fc2d..4647619 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -17,9 +17,7 @@ import yaml # Packages every agent container needs regardless of its specific business logic. -# grpcio-tools/pyyaml/ipdb/ipython aren't needed/used, but keeping to keep the scope constrained right now -# - Leave a comment if you want me to remove these, I kept them in since you originally had them but they aren't used -BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "ipdb", "ipython", "boto3"] +BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "boto3"] # Workflow will always require these BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask", "sqlalchemy", "psycopg[binary]"] @@ -322,6 +320,21 @@ def _copy_files(output_dir, files_to_copy): shutil.copy2(src, dest_path) +def _write_entrypoint_file(src, dest_path, project_dir): + """Copy an entrypoint file to dest_path, injecting a sys.path entry for its + original sibling directory so a co-located, non-stub helper import still resolves.""" + original_dir = os.path.dirname(os.path.relpath(src, project_dir)) if project_dir else "" + if not original_dir: + shutil.copy2(src, dest_path) + return + injection = ( + f"import sys, os\n" + f"sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), {original_dir!r}))\n" + ) + with open(src) as f, open(dest_path, "w") as out: + out.write(injection + f.read()) + + def generate_docker( yaml_path, agent_file, @@ -405,8 +418,7 @@ def generate_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - - files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -416,6 +428,14 @@ def generate_docker( _copy_files(output_dir, files_to_copy) + # Copy the agent's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(agent_file), + os.path.join(output_dir, os.path.basename(agent_file)), + project_dir, + ) + # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -499,7 +519,6 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ - (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), (os.path.join(script_dir, "deploy.py"), "deploy.py"), @@ -527,6 +546,7 @@ def generate_workflow_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -536,6 +556,14 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) + # Copy the workflow's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(workflow_file), + os.path.join(output_dir, workflow_basename), + project_dir, + ) + # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time From 8baed6c324536bc88c0446b16376782c5b5cd9e4 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 11:23:57 -0700 Subject: [PATCH 14/27] Parallelize per-instance polling in _poll_controllers Metrics/telemetry latency scaled with instance count x per-instance round-trip time since every instance was polled sequentially, one blocking the next, with the following tick only starting after the whole pass finished. Extracted the per-instance body into _poll_one_instance (whole body wrapped in one top-level try/except, since ThreadPoolExecutor.map() re-raises on first exception when results are consumed) and run all instances concurrently via the same ThreadPoolExecutor pattern _trigger_cleanup already used. Co-Authored-By: Claude Sonnet 5 --- ventis/controller/global_controller.py | 217 +++++++++++++------------ 1 file changed, 115 insertions(+), 102 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 16e0e9c..6584ef8 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -648,118 +648,131 @@ def _poll_controllers(self): if self.running: self.process_supervisor.check_and_respawn() - for instance in self.instance_manager.list_instances(): + # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer + # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. + instances = self.instance_manager.list_instances() + if instances: + with ThreadPoolExecutor(max_workers=len(instances)) as executor: + list(executor.map(self._poll_one_instance, instances)) + + def _poll_one_instance(self, instance): + """Poll and persist one instance's runtime/metrics/health data; never raises.""" + try: name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, + except Exception as e: + logger.warning("Failed to poll instance %s: %s", instance, e) + return + + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" + + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now - - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, - ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) - else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status + else: + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From cd3a5214099e3bdb4493f26820cafa684b8af2fd Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 13:58:37 -0700 Subject: [PATCH 15/27] cleaned up OTel Exporter --- .../portfolio/config/global_controller.yaml | 4 - tests/test_otel_exporter_fanout.py | 58 +-- tests/test_otel_exporter_fields.py | 36 +- ventis/OTLP_Exporter/DESIGN.md | 73 ++-- ventis/OTLP_Exporter/convert.py | 17 +- ventis/OTLP_Exporter/db.py | 61 +-- ventis/OTLP_Exporter/otel_exporter.py | 67 +-- ventis/controller/global_controller.py | 385 ++++++------------ 8 files changed, 216 insertions(+), 485 deletions(-) diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index edee85f..96f371c 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -91,10 +91,6 @@ otel: endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} -# - name: langfuse -# protocol: http -# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces -# headers: {} # Polling interval in seconds poll_interval: 5 diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 0691d61..96ca1b5 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -110,22 +110,10 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): ], ) - def test_build_processors_preserves_legacy_single_destination_fallback(self): - http_exporter = object() - processor = MagicMock(name="legacy_processor") - with patch.dict( - os.environ, - {"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, - clear=True, - ), patch.object( - otel_exporter, "HttpOTLPSpanExporter", return_value=http_exporter - ) as constructor, patch.object( - otel_exporter, "BatchSpanProcessor", return_value=processor - ): - result = otel_exporter._build_processors() - - self.assertEqual(result, [("legacy", processor)]) - constructor.assert_called_once_with() + def test_build_processors_raises_when_destinations_env_unset(self): + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): + otel_exporter._build_processors() def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): invalid_values = [ @@ -184,46 +172,20 @@ def test_controller_expands_env_and_builds_langfuse_basic_auth(self): ) self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") - def test_controller_env_serializes_destinations_and_keeps_legacy_mapping(self): + def test_controller_env_serializes_destinations_only(self): # Importing the controller is intentionally local: this test remains # runnable in the exporter-only environment used by the focused suite. from ventis.controller.global_controller import GlobalController destinations = self._destination_config() - env = GlobalController._otel_exporter_env( - { - "protocol": "grpc", - "endpoint": "legacy.example:4317", - "headers": {"x-tenant": "demo"}, - "destinations": destinations, - } - ) - self.assertEqual(env["OTEL_EXPORTER_OTLP_PROTOCOL"], "grpc") - self.assertEqual(env["OTEL_EXPORTER_OTLP_ENDPOINT"], "legacy.example:4317") - self.assertEqual(env["OTEL_EXPORTER_OTLP_HEADERS"], "x-tenant=demo") + env = GlobalController._otel_exporter_env({"destinations": destinations}) + self.assertEqual(set(env), {otel_exporter.DESTINATIONS_ENV}) self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) - def test_controller_rejects_invalid_destinations_before_starting_child(self): + def test_controller_env_is_none_when_otel_not_configured(self): from ventis.controller.global_controller import GlobalController - invalid_destinations = [ - [], - [{"name": "railway", "protocol": "grpc"}], - [ - {"name": "same", "protocol": "grpc", "endpoint": "one:4317"}, - { - "name": "same", - "protocol": "http/protobuf", - "endpoint": "https://two", - }, - ], - [{"name": "bad", "protocol": "smtp", "endpoint": "example"}], - ] - for destinations in invalid_destinations: - with self.subTest(destinations=destinations), self.assertRaises(ValueError): - GlobalController._otel_exporter_env( - {"destinations": destinations} - ) + self.assertIsNone(GlobalController._otel_exporter_env({})) def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) @@ -258,7 +220,6 @@ def test_send_pending_delivers_the_same_span_to_every_processor(self): otel_exporter.db, "mark_sent" ) as mark_sent: otel_exporter._processors = [("railway", first), ("langfuse", second)] - otel_exporter._processor = None otel_exporter._send_pending() first.on_end.assert_called_once() @@ -275,7 +236,6 @@ def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_fai otel_exporter.db, "mark_sent" ) as mark_sent: otel_exporter._processors = [("railway", failed), ("langfuse", remaining)] - otel_exporter._processor = None otel_exporter._send_pending() failed.on_end.assert_called_once() diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index ff8cad6..b74177d 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -17,12 +17,7 @@ def setUp(self): def tearDown(self): os.unlink(self.db_path) - def test_init_db_migrates_existing_waiting_table(self): - with sqlite3.connect(self.db_path) as conn: - conn.execute( - "CREATE TABLE waiting (future_id TEXT PRIMARY KEY, session_id TEXT NOT NULL)" - ) - + def test_init_db_creates_waiting_table_with_full_schema(self): db.init_db(self.db_path) with sqlite3.connect(self.db_path) as conn: @@ -63,17 +58,8 @@ def test_fields_are_normalized_and_added_to_span(self): span.attributes["langfuse.observation.output"], row["output"] ) - def test_split_hash_result_is_loaded_for_finished_rows(self): - class SplitHashRedis: - def hget(self, key, field): - self.request = (key, field) - return '{"recommendation": "hold"}' - - def get(self, key): - return None - + def test_error_message_is_wired_from_redis_error_field(self): db.init_db(self.db_path) - redis = SplitHashRedis() raw = { "future_id": "11112222333344445555666677778888", "request_id": "88887777666655554444333322221111", @@ -83,16 +69,24 @@ def get(self, key): "result": "", "created_at": "1.0", "finished_at": "2.0", - "failed": "0", + "failed": "1", + "error": "agent exploded", } with patch.object(db.pricing, "compute_token_cost", return_value=0.0): - db.write_waiting_rows([raw], redis_client=redis, db_path=self.db_path) + db.write_waiting_rows([raw], db_path=self.db_path) with sqlite3.connect(self.db_path) as conn: - output = conn.execute("SELECT output FROM waiting").fetchone()[0] - self.assertEqual(redis.request, (f"future:{raw['future_id']}", "result")) - self.assertEqual(json.loads(output), {"recommendation": "hold"}) + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM waiting").fetchone() + + self.assertEqual(row["error_message"], "agent exploded") + + span = convert.waiting_row_to_span(row) + self.assertEqual(span.status.description, "agent exploded") + self.assertEqual( + span.events[0].attributes["exception.message"], "agent exploded" + ) if __name__ == "__main__": diff --git a/ventis/OTLP_Exporter/DESIGN.md b/ventis/OTLP_Exporter/DESIGN.md index d433fe0..ec2957d 100644 --- a/ventis/OTLP_Exporter/DESIGN.md +++ b/ventis/OTLP_Exporter/DESIGN.md @@ -27,26 +27,22 @@ Decisions (final status): isolation from GC's core polling/health loop and independent restart, at low added complexity since SQLite is already the entire hand-off boundary between the two. - **Config**: implemented via a new `otel:` section in `global_controller.yaml` - (`protocol`/`endpoint`/`headers`), *not* by making `otel_exporter.py` itself - config-aware. `GlobalController` translates that section into the OTel SDK's own - standard env vars (`OTEL_EXPORTER_OTLP_PROTOCOL`/`_ENDPOINT`/`_HEADERS`) and passes - them to the exporter subprocess via `ProcessSupervisor.register(..., env=...)`. The - exporter still just constructs `OTLPSpanExporter()` with no explicit args (endpoint - and headers are resolved by the SDK itself from those env vars, same as always) and - reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly, to pick the gRPC vs HTTP exporter - class — the one piece of protocol selection the plain SDK classes don't do on their - own. Deliberately vendor-neutral: no backend name (Postgres, Langfuse, or otherwise) - appears anywhere in `otel_exporter.py`; the destination is 100% deploy-time config, - set once in `global_controller.yaml` and never touched by app code again. The - originally-planned `database.url` repurposing (below, kept for history) was decided - against — env-var configuration is the SDK's own idiomatic mechanism, so no - exporter-side config plumbing was added, only a GC-side YAML→env-var translation. - The initial multi-destination extension uses one `otel.destinations` list and one - independent exporter/`BatchSpanProcessor` pair per destination. gRPC and HTTP - destinations may be mixed in the same list. The legacy single-destination fields - remain supported through the original standard-environment-variable path. - Configuration is read at exporter startup; changing it requires a - GlobalController/exporter restart. + holding a `destinations` list, *not* by making `otel_exporter.py` itself + config-aware. `GlobalController` serializes that list to JSON and passes it to the + exporter subprocess as a single `VENTIS_OTEL_DESTINATIONS` env var via + `ProcessSupervisor.register(..., env=...)`. The exporter builds one independent + exporter/`BatchSpanProcessor` pair per destination, picking the gRPC vs HTTP + exporter class from each destination's `protocol` field. gRPC and HTTP destinations + may be mixed in the same list. Deliberately vendor-neutral: no backend name + (Postgres, Langfuse, or otherwise) appears anywhere in `otel_exporter.py`; the + destination is 100% deploy-time config, set once in `global_controller.yaml` and + never touched by app code again. The originally-planned `database.url` repurposing + (below, kept for history) was decided against — env-var configuration is the SDK's + own idiomatic mechanism, so no exporter-side config plumbing was added, only a + GC-side YAML→env-var translation. If `otel.destinations` is absent, GlobalController + logs that no OTel metrics collection will happen and skips starting the exporter + subprocess entirely. Configuration is read at exporter startup; changing it requires + a GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own SQLite file (`ventis/OTLP_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing @@ -77,16 +73,19 @@ otel: - name: langfuse protocol: http endpoint: https://cloud.langfuse.com/api/public/otel/v1/traces - headers: {} # e.g. Authorization: "Basic " + headers: + Authorization: Basic ${LANGFUSE_OTLP_HEADERS} # deployer pre-encodes public:secret ``` -`GlobalController._otel_exporter_env()` translates each destination into the exporter -process's destination configuration and hands it to `ProcessSupervisor.register( +`GlobalController._otel_exporter_env()` translates the `destinations` list into +`VENTIS_OTEL_DESTINATIONS` and hands it to `ProcessSupervisor.register( "otel_exporter", ..., env=...)`, which supports an `env` param (merged on top of the -parent process's own environment, not a replacement). The legacy single-destination -`protocol`/`endpoint`/`headers` form remains valid and continues through the SDK's -standard OTLP environment variables. Omitting `otel:` entirely falls back to whatever -ambient env the exporter subprocess would otherwise inherit, same as before this -change. +parent process's own environment, not a replacement). If `otel.destinations` is +absent, `_otel_exporter_env()` returns `None` and `GlobalController.__init__` skips +registering the exporter subprocess entirely, logging that no OTel metrics +collection will happen. No shape +validation is duplicated on the GlobalController side (deliberately: keep this side +simple, `otel_exporter.py` itself validates destination shape at subprocess startup, +and raises if invoked directly without `VENTIS_OTEL_DESTINATIONS` set). `otel_exporter.py` parses the destination configuration at startup and constructs the appropriate OTLP exporter for each entry (gRPC or HTTP), passing that destination's @@ -167,21 +166,7 @@ Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall the poll loop's health checks and OTel writes. -### 6. Parallelized per-instance polling (`ventis/controller/global_controller.py`) -`_poll_controllers` used to loop over every instance sequentially -- Redis reads, an -OTel sqlite write, and up to two Postgres writes per instance, one instance fully -blocking the next, with the following poll tick only starting after the whole pass -finished. Total metrics/telemetry latency scaled with instance count x round-trip -time, not the configured `poll_interval`. Fixed by extracting the per-instance body -into `_poll_one_instance` (its whole body wrapped in one top-level try/except, since -`ThreadPoolExecutor.map()` re-raises on first exception when results are consumed) -and running all instances concurrently via the same `ThreadPoolExecutor` pattern -`_trigger_cleanup` already used. Known, pre-existing, previously acknowledged in -commit `a6694d9`'s own message but never actually fixed (a same-named follow-up -branch was found to contain no real threading changes) -- see company-memory for -the investigation. - -### 7. Dependencies (all added) +### 6. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). @@ -189,8 +174,6 @@ config work, since `protocol: http` now needs that package importable). ## Known gaps (not yet built) - `WriteResult()` passes an undefined `error_message` variable to its fan-out callback, which can interrupt remote consumer propagation after the callback hash is persisted. -- Redis records failure text under `error`, but the waiting-table writer reads - `error_name`/`error_message`, so exported exception details are usually empty. - Rows are marked `sent` immediately after `BatchSpanProcessor.on_end()` accepts them, before the asynchronous OTLP export is confirmed; a later delivery failure can lose a span while leaving `sent = 1`. diff --git a/ventis/OTLP_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py index 66da972..5e6ac74 100644 --- a/ventis/OTLP_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -1,10 +1,7 @@ -"""Convert a `waiting` table row (see db.py) into an OTel ReadableSpan. +"""Converts a future into an OTel ReadableSpan. -Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects -directly instead of going through Tracer.start_span() -- there's no live tracer here, -futures already finished (sometimes in another process), so this is a historical-row -conversion, not live tracing. See ventis/OTLP_Exporter/DESIGN.md for why this deviates from -the SDK's usual advice against constructing ReadableSpan by hand. +Pure function, no I/O, no batching, no network calls. Futures already finished, so this is just a +conversion. """ from opentelemetry.sdk.trace import EXCEPTION_MESSAGE, EXCEPTION_TYPE, Event, ReadableSpan @@ -66,13 +63,7 @@ def waiting_row_to_span(row): # Model, token, agent, and cache-read usage use OTel GenAI semantic-convention # names (see https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/). - # total_cost uses gen_ai.usage.cost, which isn't yet an official semconv attribute - # but is the name Langfuse's OTel ingestion actually reads (its JSON cost_details - # attribute is currently broken -- see langfuse/langfuse#11030). Observation - # input/output use Langfuse's documented JSON-string attributes. `errors` is named - # error_count, not "errors"/"error", to avoid colliding with OTel's reserved - # error.* namespace (error.type etc.), which describes a single error, not a - # count. The remaining Ventis-specific values (project_id, server/token cost + # total_cost uses gen_ai.usage.cost. The remaining Ventis-specific values (project_id, server/token cost # breakdown, cache_hit_ratio) have no GenAI or Langfuse equivalent, so they keep # plain names. attributes = { diff --git a/ventis/OTLP_Exporter/db.py b/ventis/OTLP_Exporter/db.py index e6a1834..005ba71 100644 --- a/ventis/OTLP_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -12,21 +12,18 @@ import os import sqlite3 -from ventis.controller.utils import pricing +from ventis.controller.utils import pricing +# Will need to eventually delete dependency on this and move to OTLP +# It is currently stored here for backcompat with the old telemetry collecting + DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "otel_queue.db") -# Demo-only multipliers for scaling displayed costs; not real recorded costs. Kept -# deliberately standalone/duplicated from telemetry_logging.py's identical constants -# (rather than importing them) so this module has no dependency on it -- keep these in -# sync by hand if the multipliers there ever change. +# Demo-only multipliers for scaling displayed costs, DELETE FOR MORE ACCURATE METRICS _TOKEN_COST_MULTIPLIER = 10000 _SERVER_COST_MULTIPLIER = 100000 -# Timestamps are stored as unix epoch seconds (matching the Redis future hash fields -# they're read from), not as SQLite datetime strings. Column set mirrors -# runtime_information 1:1 (see telemetry_logging.py) plus this pipeline's own additions -# (error_name/error_message/sent). +# Table schema _TABLE_COLUMNS = """ future_id TEXT PRIMARY KEY, parent_id TEXT, @@ -55,28 +52,14 @@ name TEXT, input TEXT, output TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, sent BOOLEAN DEFAULT 0 """ -_MIGRATION_COLUMNS = { - "name": "TEXT", - "input": "TEXT", - "output": "TEXT", -} - - def init_db(db_path=DB_PATH): - """Create the waiting table and add columns missing from older databases.""" + """Create the waiting table if it doesn't already exist.""" conn = sqlite3.connect(db_path) try: conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") - existing_columns = { - row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() - } - for column, column_type in _MIGRATION_COLUMNS.items(): - if column not in existing_columns: - conn.execute(f"ALTER TABLE waiting ADD COLUMN {column} {column_type}") conn.commit() finally: conn.close() @@ -147,20 +130,16 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH part for part in (service, method) if part ) result = raw.get("result") - # Compatibility with pre-consolidation deployments, where completion - # metrics live in future:{id}:metrics but result lives in future:{id}. - # Unified hashes already include result and avoid this extra read. - if not result and finished_at and redis_client is not None: - result = redis_client.hget(f"future:{fid}", "result") - - token_cost = ( - pricing.compute_token_cost( - raw.get("model"), input_token_count, output_token_count - ) - * _TOKEN_COST_MULTIPLIER - ) - # Server cost needs an elapsed duration -- only available once finished. + + # Cost figures are only meaningful once the future has finished, so skip + # computing them until then rather than recomputing on every poll. if finished_at is not None: + token_cost = ( + pricing.compute_token_cost( + raw.get("model"), input_token_count, output_token_count + ) + * _TOKEN_COST_MULTIPLIER + ) server_cost = ( pricing.compute_server_cost( redis_client.get(f"agent:{agent_id}:instance_type") @@ -171,6 +150,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH * _SERVER_COST_MULTIPLIER ) else: + token_cost = 0.0 server_cost = 0.0 conn.execute( @@ -203,7 +183,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH "cached_tokens": cached_tokens, "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, "error_name": raw.get("error_name"), - "error_message": raw.get("error_message"), + "error_message": raw.get("error") or raw.get("error_message"), "name": name or agent_id or "unknown_agent", "input": _normalize_json_text(raw.get("args")), "output": _normalize_json_text(result), @@ -215,10 +195,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH def mark_sent(future_id, db_path=DB_PATH): - """Mark one waiting row sent. Call this immediately after successfully handing its - span to the batch processor -- one row, one commit -- so a crash between two rows' - sends can't leave an already-sent row unmarked (which would cause a duplicate send - on the next run).""" + """Mark one waiting row sent. Atomic Operation""" conn = sqlite3.connect(db_path) try: conn.execute("UPDATE waiting SET sent = 1 WHERE future_id = ?", (future_id,)) diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index cad2af8..eafb786 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,11 +5,8 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController may provide a JSON list in ``VENTIS_OTEL_DESTINATIONS``. That is a -Ventis-specific configuration because the standard OTEL exporter environment -variables describe only one destination. If it is absent, the original single -destination behavior is retained: the exporter class and its settings are selected -from the standard OTEL environment variables and SDK defaults. +GlobalController provides a JSON list in ``VENTIS_OTEL_DESTINATIONS``, required because +the standard OTEL exporter environment variables describe only one destination. """ import json @@ -35,33 +32,11 @@ logger = logging.getLogger(__name__) _running = True -_processor = None _processors = [] POLL_INTERVAL_SECONDS = 5 DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" -def _normalize_protocol(protocol): - """Return the exporter family for a configured protocol name.""" - if not isinstance(protocol, str) or not protocol.strip(): - raise ValueError("destination protocol must be a non-empty string") - normalized = protocol.strip().lower().replace("_", "-") - if normalized in {"grpc", "otlp/grpc", "grpc/protobuf", "grpc-protobuf"}: - return "grpc" - if normalized in { - "http", - "http/protobuf", - "http-protobuf", - "http/proto", - "http+protobuf", - "protobuf", - }: - return "http" - raise ValueError( - f"unsupported destination protocol {protocol!r}; expected grpc or http/protobuf" - ) - - def _validate_destination(destination, index): if not isinstance(destination, dict): raise ValueError(f"destination {index} must be an object") @@ -70,7 +45,7 @@ def _validate_destination(destination, index): if not isinstance(name, str) or not name.strip(): raise ValueError(f"destination {index} name must be a non-empty string") - protocol = _normalize_protocol(destination.get("protocol")) + protocol = destination.get("protocol") # must be exactly "grpc" or "http" endpoint = destination.get("endpoint") if not isinstance(endpoint, str) or not endpoint.strip(): raise ValueError(f"destination {name!r} endpoint must be a non-empty string") @@ -112,13 +87,7 @@ def _validate_destination(destination, index): def _configured_destinations(): - """Parse and validate the Ventis multi-destination environment variable. - - ``None`` means no Ventis-specific configuration was supplied, so callers can - preserve legacy OTEL environment-variable behavior. An empty or malformed value - is an explicit configuration error and fails startup rather than silently - exporting to the wrong destination. - """ + """Parse and validate the Ventis multi-destination environment variable.""" raw = os.environ.get(DESTINATIONS_ENV) if raw is None: return None @@ -142,18 +111,15 @@ def _configured_destinations(): def _build_exporter(destination): - """Construct one explicitly configured exporter without logging credentials.""" + """Construct one OTLP exporter.""" kwargs = { "endpoint": destination["endpoint"], } - if destination["headers"] is not None: - kwargs["headers"] = destination["headers"] - if destination["timeout"] is not None: - kwargs["timeout"] = destination["timeout"] + if destination["headers"] is not None: kwargs["headers"] = destination["headers"] # fmt: skip + if destination["timeout"] is not None: kwargs["timeout"] = destination["timeout"] # fmt: skip if destination["protocol"] == "grpc": - if destination["insecure"] is not None: - kwargs["insecure"] = destination["insecure"] + if destination["insecure"] is not None: kwargs["insecure"] = destination["insecure"] # fmt: skip return GrpcOTLPSpanExporter(**kwargs) if destination["insecure"] is not None: @@ -166,14 +132,10 @@ def _build_exporter(destination): def _build_processors(): - """Build destination processors, or one legacy processor when unconfigured.""" + """Build one exporter/BatchSpanProcessor pair per configured destination.""" destinations = _configured_destinations() if destinations is None: - protocol = os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower() - exporter_class = ( - HttpOTLPSpanExporter if protocol.startswith("http") else GrpcOTLPSpanExporter - ) - return [("legacy", BatchSpanProcessor(exporter_class(), schedule_delay_millis=1000))] + raise RuntimeError(f"{DESTINATIONS_ENV} is not set; otel.destinations is required") processors = [] try: @@ -205,10 +167,6 @@ def _handle_shutdown(signum, frame): def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" processors = _processors - if not processors and _processor is not None: - # Compatibility for callers that configured the pre-fan-out singular - # ``_processor`` directly (the normal startup path always populates both). - processors = [("legacy", _processor)] if not processors: raise RuntimeError("OTel exporter has no configured processors") @@ -256,14 +214,11 @@ def _send_pending(): def main(): - global _processor, _processors + global _processors signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() _processors = _build_processors() - # Keep the old singular module variable available to integrations that imported - # it, while all sending uses the destination-aware collection above. - _processor = _processors[0][1] logger.info("OTel exporter process started with %d destination(s).", len(_processors)) try: last_poll = 0 diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 6584ef8..ba53881 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,11 +3,8 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit -import base64 -import importlib.util import json import logging -import math import os import re import shlex @@ -19,6 +16,7 @@ from concurrent.futures import ThreadPoolExecutor import yaml +from ventis.OTLP_Exporter import db as otel_db from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs from ventis.controller.utils.env_file import resolve_env_file @@ -110,7 +108,7 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread, woken by each poll tick rather than its own timer -- see ventis/OTLP_Exporter/DESIGN.md. + # Start background cleanup thread self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() @@ -123,21 +121,19 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # Legacy `otel:` fields map straight to the OTel SDK's own standard env vars. - # A `destinations` list is additionally passed as one Ventis-specific JSON - # variable; the exporter subprocess remains a plain OTel process otherwise. + + # Passing OTel info from yaml file to process, so process doesn't have external facing logic otel_env = self._otel_exporter_env(self.config.get("otel", {})) - self.process_supervisor.register( - "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env - ) + if otel_env is not None: + self.process_supervisor.register( + "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + ) + else: + logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") # Initialize/migrate the waiting table synchronously before either the GC or # exporter process can access it. - otel_db_spec = importlib.util.spec_from_file_location( - "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") - ) - self._otel_db = importlib.util.module_from_spec(otel_db_spec) - otel_db_spec.loader.exec_module(self._otel_db) + self._otel_db = otel_db self._otel_db.init_db() self.process_supervisor.start_all() @@ -192,10 +188,7 @@ def _load_config(config_path): GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: config = yaml.safe_load(f) - if "ec2" in config: - config["ec2"] = GlobalController._expand_env_value(config["ec2"]) - if "database" in config: - config["database"] = GlobalController._expand_env_value(config["database"]) + config = GlobalController._expand_env_value(config) return config @staticmethod @@ -228,125 +221,21 @@ def _expand_env_value(value): @staticmethod def _otel_exporter_env(otel_cfg): - """Translate global_controller.yaml's `otel:` section into standard OTLP env - vars for the exporter subprocess. When present, ``destinations`` is passed as - JSON for the exporter to construct a fan-out. Returns None if `otel:` is - absent/empty so the subprocess falls back to the SDK's own defaults untouched. - - The legacy protocol/endpoint/headers mappings intentionally remain unchanged - for existing configurations. - """ - env = {} - if otel_cfg.get("protocol"): - env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] - if otel_cfg.get("endpoint"): - env["OTEL_EXPORTER_OTLP_ENDPOINT"] = otel_cfg["endpoint"] - if otel_cfg.get("headers"): - env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( - f"{k}={v}" for k, v in otel_cfg["headers"].items() - ) - - if "destinations" in otel_cfg: - destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) - for destination in destinations: - if destination.get("name") == "langfuse": - public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") - secret_key = os.environ.get("LANGFUSE_SECRET_KEY") - if public_key and secret_key: - auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode() - destination.setdefault("headers", {})["Authorization"] = f"Basic {auth}" - GlobalController._validate_otel_destinations(destinations) - try: - env["VENTIS_OTEL_DESTINATIONS"] = json.dumps(destinations) - except (TypeError, ValueError) as exc: - # Do not include the offending value: destination configs commonly - # contain credentials in headers. - raise ValueError( - "otel.destinations must contain JSON-serializable values" - ) from exc - return env or None - - @staticmethod - def _validate_otel_destinations(destinations): - """Validate the shape of the optional exporter fan-out configuration. - - Keep this validation deliberately structural: destination-specific options - are interpreted by the exporter. Error messages identify only the location - and type, never destination contents or header values. + """Translate global_controller.yaml's `otel:` section into the exporter + subprocess's env. Returns None if `otel.destinations` is absent, so the + caller skips starting the exporter subprocess entirely. Destination + shape/protocol is validated by the exporter subprocess itself + (otel_exporter.py), not duplicated here. """ - if not isinstance(destinations, list): - raise ValueError("otel.destinations must be a list") - if not destinations: - raise ValueError("otel.destinations must not be empty") - - names = set() - for index, destination in enumerate(destinations): - if not isinstance(destination, dict): - raise ValueError( - f"otel.destinations[{index}] must be a mapping" - ) - - for field in ("name", "protocol", "endpoint"): - value = destination.get(field) - if not isinstance(value, str) or not value.strip(): - raise ValueError( - f"otel.destinations[{index}].{field} must be a non-empty string" - ) - - name = destination["name"].strip() - if name in names: - raise ValueError(f"otel.destinations contains duplicate name {name!r}") - names.add(name) - - protocol = destination["protocol"].strip().lower().replace("_", "-") - if protocol not in { - "grpc", - "otlp/grpc", - "grpc/protobuf", - "grpc-protobuf", - "http", - "http/protobuf", - "http-protobuf", - "http/proto", - "http+protobuf", - "protobuf", - }: - raise ValueError( - f"otel.destinations[{index}].protocol must be grpc or http/protobuf" - ) - - if "headers" in destination: - headers = destination["headers"] - if not isinstance(headers, dict): - raise ValueError( - f"otel.destinations[{index}].headers must be a mapping" - ) - if any( - not isinstance(key, str) or not isinstance(value, str) - for key, value in headers.items() - ): - raise ValueError( - f"otel.destinations[{index}].headers keys and values must be strings" - ) - - if "insecure" in destination and not isinstance( - destination["insecure"], bool - ): - raise ValueError( - f"otel.destinations[{index}].insecure must be a boolean" - ) - - if "timeout" in destination: - timeout = destination["timeout"] - if ( - isinstance(timeout, bool) - or not isinstance(timeout, (int, float)) - or not math.isfinite(timeout) - or timeout <= 0 - ): - raise ValueError( - f"otel.destinations[{index}].timeout must be a positive number" - ) + if "destinations" not in otel_cfg: + return None + destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) + try: + return {"VENTIS_OTEL_DESTINATIONS": json.dumps(destinations)} + except (TypeError, ValueError) as exc: + raise ValueError( + "otel.destinations must contain JSON-serializable values" + ) from exc @staticmethod def _get_replica_placements(ctrl): @@ -641,138 +530,124 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ - # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which - # terminates every managed process) via the signal handler before this line is - # reached -- without the guard, this could respawn a process just intentionally - # killed. See ventis/OTLP_Exporter/DESIGN.md. + # Prevents a process from restarting if a deliberate kill-cmd happens if self.running: self.process_supervisor.check_and_respawn() - # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer - # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. - instances = self.instance_manager.list_instances() - if instances: - with ThreadPoolExecutor(max_workers=len(instances)) as executor: - list(executor.map(self._poll_one_instance, instances)) - - def _poll_one_instance(self, instance): - """Poll and persist one instance's runtime/metrics/health data; never raises.""" - try: + for instance in self.instance_manager.list_instances(): name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - except Exception as e: - logger.warning("Failed to poll instance %s: %s", instance, e) - return - - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + # This is now legacy, keeping it for now, but will remove this later + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now + + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status - else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) + else: + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From 1e6a6d8d3aad5e692b00a76c7cf9ea8db2d445d0 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 14:25:29 -0700 Subject: [PATCH 16/27] Align with feature/otel-exporter: use updated langfuse config example and remove _write_entrypoint_file - Fixed langfuse example to use generic env-var headers pattern - Removed _write_entrypoint_file (directory structure preservation via _sweep_py_files is cleaner) --- .../portfolio/config/global_controller.yaml | 4 -- ventis/stub_generator.py | 40 +++---------------- 2 files changed, 6 insertions(+), 38 deletions(-) diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index edee85f..96f371c 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -91,10 +91,6 @@ otel: endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} -# - name: langfuse -# protocol: http -# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces -# headers: {} # Polling interval in seconds poll_interval: 5 diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 9fe38a5..33824ff 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -17,7 +17,9 @@ import yaml # Packages every agent container needs regardless of its specific business logic. -BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "boto3"] +# grpcio-tools/pyyaml/ipdb/ipython aren't needed/used, but keeping to keep the scope constrained right now +# - Leave a comment if you want me to remove these, I kept them in since you originally had them but they aren't used +BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "ipdb", "ipython", "boto3"] # Workflow will always require these BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask", "sqlalchemy", "psycopg[binary]"] @@ -321,21 +323,6 @@ def _copy_files(output_dir, files_to_copy): shutil.copy2(src, dest_path) -def _write_entrypoint_file(src, dest_path, project_dir): - """Copy an entrypoint file to dest_path, injecting a sys.path entry for its - original sibling directory so a co-located, non-stub helper import still resolves.""" - original_dir = os.path.dirname(os.path.relpath(src, project_dir)) if project_dir else "" - if not original_dir: - shutil.copy2(src, dest_path) - return - injection = ( - f"import sys, os\n" - f"sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), {original_dir!r}))\n" - ) - with open(src) as f, open(dest_path, "w") as out: - out.write(injection + f.read()) - - def generate_docker( yaml_path, agent_file, @@ -419,7 +406,8 @@ def generate_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) + + files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -429,14 +417,6 @@ def generate_docker( _copy_files(output_dir, files_to_copy) - # Copy the agent's own real file last, so it wins over any same-named stub - # copy above; inject a sys.path entry so its own sibling helpers still resolve. - _write_entrypoint_file( - os.path.abspath(agent_file), - os.path.join(output_dir, os.path.basename(agent_file)), - project_dir, - ) - # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -520,6 +500,7 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ + (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), (os.path.join(script_dir, "deploy.py"), "deploy.py"), @@ -547,7 +528,6 @@ def generate_workflow_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -557,14 +537,6 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) - # Copy the workflow's own real file last, so it wins over any same-named stub - # copy above; inject a sys.path entry so its own sibling helpers still resolve. - _write_entrypoint_file( - os.path.abspath(workflow_file), - os.path.join(output_dir, workflow_basename), - project_dir, - ) - # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time From ede3facbdcc4ed3ac64ffb3b72d621609b9a74f7 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 14:27:17 -0700 Subject: [PATCH 17/27] Restore parallelized polling implementation Re-applied the parallel instance polling that was lost during conflict resolution. The _poll_controllers method now uses ThreadPoolExecutor to poll all instances concurrently via _poll_one_instance, preventing one slow instance's Redis/Postgres round-trip from blocking the entire poll tick. --- ventis/controller/global_controller.py | 222 +++++++++++++------------ 1 file changed, 118 insertions(+), 104 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index ba53881..b6b3948 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -530,124 +530,138 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ - # Prevents a process from restarting if a deliberate kill-cmd happens + # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which + # terminates every managed process) via the signal handler before this line is + # reached -- without the guard, this could respawn a process just intentionally + # killed. See ventis/OTLP_Exporter/DESIGN.md. if self.running: self.process_supervisor.check_and_respawn() - for instance in self.instance_manager.list_instances(): + # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer + # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. + instances = self.instance_manager.list_instances() + if instances: + with ThreadPoolExecutor(max_workers=len(instances)) as executor: + list(executor.map(self._poll_one_instance, instances)) + + def _poll_one_instance(self, instance): + """Poll and persist one instance's runtime/metrics/health data; never raises.""" + try: name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) + except Exception as e: + logger.warning("Failed to poll instance %s: %s", instance, e) + return - # This is now legacy, keeping it for now, but will remove this later - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" + + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now - - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, - ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) - else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status + else: + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From ea91ff9f776014bd85ef4ef0417f01d07f3d93e6 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 14:32:42 -0700 Subject: [PATCH 18/27] added concurrent polling --- ventis/controller/global_controller.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index b6b3948..6bdbd1a 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -530,10 +530,7 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ - # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which - # terminates every managed process) via the signal handler before this line is - # reached -- without the guard, this could respawn a process just intentionally - # killed. See ventis/OTLP_Exporter/DESIGN.md. + # Prevents a process from restarting if a deliberate kill-cmd happens if self.running: self.process_supervisor.check_and_respawn() @@ -560,6 +557,7 @@ def _poll_one_instance(self, instance): self._otel_db.write_waiting_rows( future_rows, node_redis, self.config.get("project_id", 0) ) + # This is now legacy, keeping it for now, but will remove this later send_runtime_information( future_rows, node_redis, From b4e45e8bcf4ea516123b8a7eab68c936637c42ad Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:32:00 -0700 Subject: [PATCH 19/27] Redis-backed otel destination reload: config change no longer needs full redeploy Moves otel.destinations from a one-shot VENTIS_OTEL_DESTINATIONS env var (frozen at exporter subprocess spawn) to a Redis key (otel:destinations), mirroring the existing routing-table live-reload pattern. GlobalController writes it at startup and again in reload_config() (SIGHUP); otel_exporter.py's existing 5s poll tick re-reads it each cycle and rebuilds its BatchSpanProcessors only when it changed. No signal-forwarding, no subprocess restart, no ProcessSupervisor.restart -- just a small ProcessSupervisor.is_registered() so reload_config knows whether the exporter is even running. Kept in scope: exporter start-gating at boot is unchanged (still skipped entirely if otel.destinations is absent at startup); destinations added after boot only take effect if the exporter was already running. --- tests/test_otel_exporter_fanout.py | 164 +++++++++++++----- ventis/OTLP_Exporter/otel_exporter.py | 69 ++++++-- ventis/controller/global_controller.py | 57 ++++-- ventis/controller/utils/process_supervisor.py | 5 + 4 files changed, 230 insertions(+), 65 deletions(-) diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 96ca1b5..5231115 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -69,11 +69,7 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): http_processor = MagicMock(name="http_processor") destinations = self._destination_config() - with patch.dict( - os.environ, - {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, - clear=True, - ), patch.object( + with patch.object( otel_exporter, "GrpcOTLPSpanExporter", return_value=grpc_exporter, @@ -86,7 +82,7 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): "BatchSpanProcessor", side_effect=[grpc_processor, http_processor], ) as processor_constructor: - processors = otel_exporter._build_processors() + processors = otel_exporter._build_processors(json.dumps(destinations)) self.assertEqual( processors, [("railway", grpc_processor), ("langfuse", http_processor)] @@ -110,10 +106,9 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): ], ) - def test_build_processors_raises_when_destinations_env_unset(self): - with patch.dict(os.environ, {}, clear=True): - with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): - otel_exporter._build_processors() + def test_build_processors_raises_when_destinations_raw_is_none(self): + with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): + otel_exporter._build_processors(None) def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): invalid_values = [ @@ -135,25 +130,23 @@ def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(se ), ] for raw in invalid_values: - with self.subTest(raw=raw), patch.dict( - os.environ, {otel_exporter.DESTINATIONS_ENV: raw}, clear=True - ): + with self.subTest(raw=raw): with self.assertRaises(ValueError): - otel_exporter._configured_destinations() + otel_exporter._configured_destinations(raw) - def test_controller_expands_env_and_builds_langfuse_basic_auth(self): + def test_controller_expands_env_in_destinations(self): + # NOTE: the pre-existing Basic-auth-header-injection expectation this test + # once carried was already unimplemented/failing before the Redis-backed + # reload change (VENTIS_OTEL_DESTINATIONS -> otel:destinations); out of + # scope here, so this only covers ${ENV_VAR} expansion, which does work. from ventis.controller.global_controller import GlobalController with patch.dict( os.environ, - { - "LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com", - "LANGFUSE_PUBLIC_KEY": "public", - "LANGFUSE_SECRET_KEY": "secret", - }, + {"LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com"}, clear=True, ): - env = GlobalController._otel_exporter_env( + destinations = GlobalController._otel_destinations( { "destinations": [ { @@ -165,27 +158,45 @@ def test_controller_expands_env_and_builds_langfuse_basic_auth(self): } ) - destination = json.loads(env[otel_exporter.DESTINATIONS_ENV])[0] self.assertEqual( - destination["endpoint"], + destinations[0]["endpoint"], "https://us.cloud.langfuse.com/api/public/otel/v1/traces", ) - self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") - def test_controller_env_serializes_destinations_only(self): - # Importing the controller is intentionally local: this test remains - # runnable in the exporter-only environment used by the focused suite. + def test_controller_destinations_is_none_when_otel_not_configured(self): from ventis.controller.global_controller import GlobalController - destinations = self._destination_config() - env = GlobalController._otel_exporter_env({"destinations": destinations}) - self.assertEqual(set(env), {otel_exporter.DESTINATIONS_ENV}) - self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) + self.assertIsNone(GlobalController._otel_destinations({})) - def test_controller_env_is_none_when_otel_not_configured(self): + def test_controller_exporter_env_carries_redis_connection_only(self): + # Destinations travel via Redis (otel:destinations), not env, so this + # is now just the fixed connection info the subprocess needs to reach it. from ventis.controller.global_controller import GlobalController - self.assertIsNone(GlobalController._otel_exporter_env({})) + env = GlobalController._otel_exporter_env( + {"host": "redis-host", "port": 6380, "db": 2} + ) + self.assertEqual( + env, + { + "VENTIS_REDIS_HOST": "redis-host", + "VENTIS_REDIS_PORT": "6380", + "VENTIS_REDIS_DB": "2", + }, + ) + + def test_controller_exporter_env_defaults(self): + from ventis.controller.global_controller import GlobalController + + env = GlobalController._otel_exporter_env({}) + self.assertEqual( + env, + { + "VENTIS_REDIS_HOST": "localhost", + "VENTIS_REDIS_PORT": "6379", + "VENTIS_REDIS_DB": "0", + }, + ) def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) @@ -251,11 +262,7 @@ def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_fai def test_processor_construction_failure_shuts_down_already_built_processors(self): first_processor = MagicMock(name="first_processor") destinations = self._destination_config() - with patch.dict( - os.environ, - {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, - clear=True, - ), patch.object( + with patch.object( otel_exporter, "GrpcOTLPSpanExporter", return_value=object(), @@ -269,10 +276,89 @@ def test_processor_construction_failure_shuts_down_already_built_processors(self return_value=first_processor, ): with self.assertRaisesRegex(RuntimeError, "bad HTTP exporter"): - otel_exporter._build_processors() + otel_exporter._build_processors(json.dumps(destinations)) first_processor.shutdown.assert_called_once_with() +class OTelExporterReloadTests(unittest.TestCase): + """Redis-backed live reload: each poll tick re-reads otel:destinations and + rebuilds _processors only when it changed.""" + + def setUp(self): + self._orig_redis = otel_exporter._redis + self._orig_raw = otel_exporter._last_destinations_raw + self._orig_processors = otel_exporter._processors + self.store = {} + + class FakeRedis: + def get(_self, key): + return self.store.get(key) + + otel_exporter._redis = FakeRedis() + otel_exporter._last_destinations_raw = None + otel_exporter._processors = [] + + def tearDown(self): + otel_exporter._redis = self._orig_redis + otel_exporter._last_destinations_raw = self._orig_raw + otel_exporter._processors = self._orig_processors + + def test_reload_builds_processors_from_redis_on_first_read(self): + destinations = self._config_for("a", "grpc") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(destinations) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=MagicMock(name="p") + ): + otel_exporter._reload_destinations_if_changed() + self.assertEqual([name for name, _ in otel_exporter._processors], ["a"]) + + def test_reload_is_a_noop_when_redis_value_is_unchanged(self): + destinations = self._config_for("a", "grpc") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(destinations) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=MagicMock(name="p") + ) as processor_ctor: + otel_exporter._reload_destinations_if_changed() + otel_exporter._reload_destinations_if_changed() + processor_ctor.assert_called_once() + + def test_reload_rebuilds_and_shuts_down_old_processors_when_redis_value_changes(self): + old_processor = MagicMock(name="old") + new_processor = MagicMock(name="new") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("a", "grpc")) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=old_processor + ): + otel_exporter._reload_destinations_if_changed() + + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("b", "http")) + with patch.object(otel_exporter, "HttpOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=new_processor + ): + otel_exporter._reload_destinations_if_changed() + + old_processor.shutdown.assert_called_once_with() + self.assertEqual([name for name, _ in otel_exporter._processors], ["b"]) + + def test_reload_keeps_previous_processors_when_new_redis_value_is_invalid(self): + good = MagicMock(name="good") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("a", "grpc")) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=good + ): + otel_exporter._reload_destinations_if_changed() + + self.store[otel_exporter.DESTINATIONS_KEY] = "not json" + otel_exporter._reload_destinations_if_changed() + + good.shutdown.assert_not_called() + self.assertEqual([name for name, _ in otel_exporter._processors], ["a"]) + + @staticmethod + def _config_for(name, protocol): + return [{"name": name, "protocol": protocol, "endpoint": "host:1"}] + + if __name__ == "__main__": unittest.main() diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index eafb786..94c195e 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,8 +5,12 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController provides a JSON list in ``VENTIS_OTEL_DESTINATIONS``, required because -the standard OTEL exporter environment variables describe only one destination. +GlobalController writes the resolved destination list to the ``otel:destinations`` Redis +key (required because the standard OTEL exporter environment variables describe only one +destination). Every poll tick also re-reads that key and rebuilds the configured +processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) +reaches this process without a restart. Redis connection info itself, unlike +destinations, is fixed for the process's lifetime and passed once via env. """ import json @@ -15,8 +19,12 @@ import os import signal import sqlite3 +import sys import time +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from ventis.utils.redis_client import RedisClient + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter as GrpcOTLPSpanExporter, ) @@ -33,8 +41,20 @@ _running = True _processors = [] +_last_destinations_raw = None POLL_INTERVAL_SECONDS = 5 -DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" +DESTINATIONS_KEY = "otel:destinations" # keep in sync with GlobalController.OTEL_DESTINATIONS_KEY + + +def _redis_client(): + return RedisClient( + host=os.environ.get("VENTIS_REDIS_HOST", "localhost"), + port=int(os.environ.get("VENTIS_REDIS_PORT", 6379)), + db=int(os.environ.get("VENTIS_REDIS_DB", 0)), + ) + + +_redis = None def _validate_destination(destination, index): @@ -86,17 +106,16 @@ def _validate_destination(destination, index): } -def _configured_destinations(): - """Parse and validate the Ventis multi-destination environment variable.""" - raw = os.environ.get(DESTINATIONS_ENV) +def _configured_destinations(raw): + """Parse and validate the destinations JSON read from Redis.""" if raw is None: return None try: destinations = json.loads(raw) except (TypeError, json.JSONDecodeError) as exc: - raise ValueError(f"{DESTINATIONS_ENV} must contain a JSON list") from exc + raise ValueError(f"{DESTINATIONS_KEY} must contain a JSON list") from exc if not isinstance(destinations, list) or not destinations: - raise ValueError(f"{DESTINATIONS_ENV} must contain a non-empty JSON list") + raise ValueError(f"{DESTINATIONS_KEY} must contain a non-empty JSON list") validated = [] names = set() @@ -131,11 +150,11 @@ def _build_exporter(destination): return HttpOTLPSpanExporter(**kwargs) -def _build_processors(): +def _build_processors(raw): """Build one exporter/BatchSpanProcessor pair per configured destination.""" - destinations = _configured_destinations() + destinations = _configured_destinations(raw) if destinations is None: - raise RuntimeError(f"{DESTINATIONS_ENV} is not set; otel.destinations is required") + raise RuntimeError(f"{DESTINATIONS_KEY} is not set; otel.destinations is required") processors = [] try: @@ -164,6 +183,27 @@ def _handle_shutdown(signum, frame): _running = False +def _reload_destinations_if_changed(): + """Re-read otel:destinations from Redis; rebuild _processors if it changed. + Invalid or missing values are logged and the previous processors are kept + running, matching the poll loop's existing non-fatal error handling. + """ + global _processors, _last_destinations_raw + raw = _redis.get(DESTINATIONS_KEY) + if raw == _last_destinations_raw: + return + try: + new_processors = _build_processors(raw) + except Exception as e: + logger.warning("Ignoring invalid %s update: %s", DESTINATIONS_KEY, e) + return + for _, processor in _processors: + processor.shutdown() + _processors = new_processors + _last_destinations_raw = raw + logger.info("Reloaded %d OTel destination(s) from Redis.", len(_processors)) + + def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" processors = _processors @@ -214,17 +254,20 @@ def _send_pending(): def main(): - global _processors + global _processors, _redis, _last_destinations_raw signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _processors = _build_processors() + _redis = _redis_client() + _last_destinations_raw = _redis.get(DESTINATIONS_KEY) + _processors = _build_processors(_last_destinations_raw) logger.info("OTel exporter process started with %d destination(s).", len(_processors)) try: last_poll = 0 while _running: if time.time() - last_poll >= POLL_INTERVAL_SECONDS: try: + _reload_destinations_if_changed() _send_pending() except Exception as e: logger.warning("Poll cycle failed (non-fatal): %s", e) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 6bdbd1a..7bd1a5d 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -65,6 +65,7 @@ class GlobalController(object): SERVICES_SET_KEY = "routing_table:services" POLICY_RULES_KEY = "policy:rules" IDENTITY_KEY = "controller:identity" # has controllers current project_id and database_url + OTEL_DESTINATIONS_KEY = "otel:destinations" # otel_exporter subprocess polls this to pick up config changes def __init__(self, config_path): self.config_path = config_path @@ -121,12 +122,17 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - - # Passing OTel info from yaml file to process, so process doesn't have external facing logic - otel_env = self._otel_exporter_env(self.config.get("otel", {})) - if otel_env is not None: + + # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter + # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) + # reach it without a restart. Only the fixed Redis connection info is passed as env. + destinations = self._otel_destinations(self.config.get("otel", {})) + if destinations is not None: + self._write_otel_destinations(destinations) self.process_supervisor.register( - "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + "otel_exporter", + [sys.executable, otel_exporter_script], + env=self._otel_exporter_env(redis_cfg), ) else: logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") @@ -220,22 +226,40 @@ def _expand_env_value(value): return value @staticmethod - def _otel_exporter_env(otel_cfg): - """Translate global_controller.yaml's `otel:` section into the exporter - subprocess's env. Returns None if `otel.destinations` is absent, so the - caller skips starting the exporter subprocess entirely. Destination - shape/protocol is validated by the exporter subprocess itself - (otel_exporter.py), not duplicated here. + def _otel_destinations(otel_cfg): + """Resolve global_controller.yaml's `otel.destinations` (expanding any + ${ENV_VAR} refs). Returns None if absent, so the caller skips starting + the exporter subprocess entirely. Destination shape/protocol is + validated by the exporter subprocess itself (otel_exporter.py), not + duplicated here. """ if "destinations" not in otel_cfg: return None - destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) + return GlobalController._expand_env_value(otel_cfg["destinations"]) + + @staticmethod + def _otel_exporter_env(redis_cfg): + """Env for the exporter subprocess: just enough to reach the same Redis + as this GlobalController. Fixed for the process's lifetime -- unlike + destinations, the Redis location itself isn't something a running + deploy can be reconfigured onto. + """ + return { + "VENTIS_REDIS_HOST": str(redis_cfg.get("host", "localhost")), + "VENTIS_REDIS_PORT": str(redis_cfg.get("port", 6379)), + "VENTIS_REDIS_DB": str(redis_cfg.get("db", 0)), + } + + def _write_otel_destinations(self, destinations): + """Push the resolved destination list to Redis. Raises if it can't be + JSON-serialized -- same validation the old env-var path had.""" try: - return {"VENTIS_OTEL_DESTINATIONS": json.dumps(destinations)} + payload = json.dumps(destinations) except (TypeError, ValueError) as exc: raise ValueError( "otel.destinations must contain JSON-serializable values" ) from exc + self.redis.set(self.OTEL_DESTINATIONS_KEY, payload) @staticmethod def _get_replica_placements(ctrl): @@ -264,6 +288,13 @@ def reload_config(self): self._write_identity() self.instance_manager.publish_routing_snapshot(self.controllers) + # Refresh otel destinations too, same as the routing table above. Only + # meaningful if the exporter subprocess is already running (started at + # boot) -- it isn't spawned mid-run just because otel got added here. + destinations = self._otel_destinations(self.config.get("otel", {})) + if destinations is not None and self.process_supervisor.is_registered("otel_exporter"): + self._write_otel_destinations(destinations) + def _write_resource_specs(self): """Write the per-agent resource specs to Redis.""" for ctrl in self.controllers: diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 8c061bc..44233c1 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -19,6 +19,11 @@ def __init__(self): self._specs = {} # name -> (argv, env) tuple self._procs = {} # name -> subprocess.Popen + def is_registered(self, name): + """Whether `name` was ever registered (regardless of whether it's still + running -- see check_and_respawn for restarts).""" + return name in self._specs + def register(self, name, argv, env=None): """Declare a process to manage. Does not start it -- call start_all() once everything is registered. `env`, if given, is merged on top of (not a From 9f630fe163cedcaba54828d2839ea5876797a5db Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:54:28 -0700 Subject: [PATCH 20/27] Simplify: assume otel_exporter's Redis is always localhost:6379 Drops the redis-connection-info env plumbing (VENTIS_REDIS_HOST/PORT/DB, GlobalController._otel_exporter_env) added in the previous commit -- otel_exporter and GlobalController always run on the same host, and RedisClient's own defaults already are localhost:6379/db0, so passing them through was dead flexibility for a case that doesn't exist yet. --- tests/test_otel_exporter_fanout.py | 30 -------------------------- ventis/OTLP_Exporter/otel_exporter.py | 16 +++----------- ventis/controller/global_controller.py | 20 +++-------------- 3 files changed, 6 insertions(+), 60 deletions(-) diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 5231115..59d4345 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -168,36 +168,6 @@ def test_controller_destinations_is_none_when_otel_not_configured(self): self.assertIsNone(GlobalController._otel_destinations({})) - def test_controller_exporter_env_carries_redis_connection_only(self): - # Destinations travel via Redis (otel:destinations), not env, so this - # is now just the fixed connection info the subprocess needs to reach it. - from ventis.controller.global_controller import GlobalController - - env = GlobalController._otel_exporter_env( - {"host": "redis-host", "port": 6380, "db": 2} - ) - self.assertEqual( - env, - { - "VENTIS_REDIS_HOST": "redis-host", - "VENTIS_REDIS_PORT": "6380", - "VENTIS_REDIS_DB": "2", - }, - ) - - def test_controller_exporter_env_defaults(self): - from ventis.controller.global_controller import GlobalController - - env = GlobalController._otel_exporter_env({}) - self.assertEqual( - env, - { - "VENTIS_REDIS_HOST": "localhost", - "VENTIS_REDIS_PORT": "6379", - "VENTIS_REDIS_DB": "0", - }, - ) - def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) try: diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index 94c195e..58704b0 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -9,8 +9,8 @@ key (required because the standard OTEL exporter environment variables describe only one destination). Every poll tick also re-reads that key and rebuilds the configured processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) -reaches this process without a restart. Redis connection info itself, unlike -destinations, is fixed for the process's lifetime and passed once via env. +reaches this process without a restart. Redis itself is assumed to be on localhost:6379, +same as GlobalController's own default -- both run on the same host. """ import json @@ -44,16 +44,6 @@ _last_destinations_raw = None POLL_INTERVAL_SECONDS = 5 DESTINATIONS_KEY = "otel:destinations" # keep in sync with GlobalController.OTEL_DESTINATIONS_KEY - - -def _redis_client(): - return RedisClient( - host=os.environ.get("VENTIS_REDIS_HOST", "localhost"), - port=int(os.environ.get("VENTIS_REDIS_PORT", 6379)), - db=int(os.environ.get("VENTIS_REDIS_DB", 0)), - ) - - _redis = None @@ -258,7 +248,7 @@ def main(): signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _redis = _redis_client() + _redis = RedisClient() # localhost:6379, db 0 -- same host as GlobalController _last_destinations_raw = _redis.get(DESTINATIONS_KEY) _processors = _build_processors(_last_destinations_raw) logger.info("OTel exporter process started with %d destination(s).", len(_processors)) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 7bd1a5d..85d2e23 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -125,14 +125,13 @@ def __init__(self, config_path): # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) - # reach it without a restart. Only the fixed Redis connection info is passed as env. + # reach it without a restart. Redis itself is assumed to be localhost:6379 (the + # exporter's own RedisClient default) -- no connection env needed. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None: self._write_otel_destinations(destinations) self.process_supervisor.register( - "otel_exporter", - [sys.executable, otel_exporter_script], - env=self._otel_exporter_env(redis_cfg), + "otel_exporter", [sys.executable, otel_exporter_script] ) else: logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") @@ -237,19 +236,6 @@ def _otel_destinations(otel_cfg): return None return GlobalController._expand_env_value(otel_cfg["destinations"]) - @staticmethod - def _otel_exporter_env(redis_cfg): - """Env for the exporter subprocess: just enough to reach the same Redis - as this GlobalController. Fixed for the process's lifetime -- unlike - destinations, the Redis location itself isn't something a running - deploy can be reconfigured onto. - """ - return { - "VENTIS_REDIS_HOST": str(redis_cfg.get("host", "localhost")), - "VENTIS_REDIS_PORT": str(redis_cfg.get("port", 6379)), - "VENTIS_REDIS_DB": str(redis_cfg.get("db", 0)), - } - def _write_otel_destinations(self, destinations): """Push the resolved destination list to Redis. Raises if it can't be JSON-serialized -- same validation the old env-var path had.""" From c085dbc86649d14f06eaa4cc787485aa7b71bcd5 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:57:17 -0700 Subject: [PATCH 21/27] Trim explanatory comments off simple/obvious functions Kept comments only where behavior is genuinely non-obvious (why the exporter polls Redis instead of restarting, why reload_config gates on is_registered, the invalid-update-keeps-old-processors fallback). Dropped comments/docstrings that just narrated 'this was added' on trivial pass-through code. --- ventis/OTLP_Exporter/otel_exporter.py | 15 +++++--------- ventis/controller/global_controller.py | 20 +++++-------------- ventis/controller/utils/process_supervisor.py | 2 -- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index 58704b0..7e365b0 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,12 +5,9 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController writes the resolved destination list to the ``otel:destinations`` Redis -key (required because the standard OTEL exporter environment variables describe only one -destination). Every poll tick also re-reads that key and rebuilds the configured -processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) -reaches this process without a restart. Redis itself is assumed to be on localhost:6379, -same as GlobalController's own default -- both run on the same host. +Destinations come from the ``otel:destinations`` Redis key (GlobalController writes it), +not env -- every poll tick re-reads it and rebuilds processors if it changed, so a config +reload (SIGHUP) reaches this process without a restart. """ import json @@ -174,10 +171,8 @@ def _handle_shutdown(signum, frame): def _reload_destinations_if_changed(): - """Re-read otel:destinations from Redis; rebuild _processors if it changed. - Invalid or missing values are logged and the previous processors are kept - running, matching the poll loop's existing non-fatal error handling. - """ + # Invalid Redis values are logged and ignored -- keep the previous processors + # running rather than tearing down a working config over a bad update. global _processors, _last_destinations_raw raw = _redis.get(DESTINATIONS_KEY) if raw == _last_destinations_raw: diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 85d2e23..fc5dd38 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -123,10 +123,8 @@ def __init__(self, config_path): otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter - # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) - # reach it without a restart. Redis itself is assumed to be localhost:6379 (the - # exporter's own RedisClient default) -- no connection env needed. + # Exporter polls self.OTEL_DESTINATIONS_KEY in Redis each cycle instead of + # reading env once, so reload_config() can update it without a restart. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None: self._write_otel_destinations(destinations) @@ -226,19 +224,12 @@ def _expand_env_value(value): @staticmethod def _otel_destinations(otel_cfg): - """Resolve global_controller.yaml's `otel.destinations` (expanding any - ${ENV_VAR} refs). Returns None if absent, so the caller skips starting - the exporter subprocess entirely. Destination shape/protocol is - validated by the exporter subprocess itself (otel_exporter.py), not - duplicated here. - """ + """Resolve otel.destinations (${ENV_VAR} refs expanded), or None if absent.""" if "destinations" not in otel_cfg: return None return GlobalController._expand_env_value(otel_cfg["destinations"]) def _write_otel_destinations(self, destinations): - """Push the resolved destination list to Redis. Raises if it can't be - JSON-serialized -- same validation the old env-var path had.""" try: payload = json.dumps(destinations) except (TypeError, ValueError) as exc: @@ -274,9 +265,8 @@ def reload_config(self): self._write_identity() self.instance_manager.publish_routing_snapshot(self.controllers) - # Refresh otel destinations too, same as the routing table above. Only - # meaningful if the exporter subprocess is already running (started at - # boot) -- it isn't spawned mid-run just because otel got added here. + # Only meaningful if the exporter was already running -- otel isn't + # spawned mid-run just because it got added to the config here. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None and self.process_supervisor.is_registered("otel_exporter"): self._write_otel_destinations(destinations) diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 44233c1..f5336e6 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -20,8 +20,6 @@ def __init__(self): self._procs = {} # name -> subprocess.Popen def is_registered(self, name): - """Whether `name` was ever registered (regardless of whether it's still - running -- see check_and_respawn for restarts).""" return name in self._specs def register(self, name, argv, env=None): From 53a96c772a6f2db822d6eb287d1397ed7e144afc Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 11:11:57 -0700 Subject: [PATCH 22/27] Ventis fixes extracted from the CLI packaging work Everything under ventis/ and tests/ that the canyonos CLI branch depends on, lifted off feature/config-reloading with no cli/ or examples/ changes. - Package layout: move deploy/future/ventis_context/bedrock/utils under ventis/controller/, add ventis/Dockerfile and ventis/README.md. - ventis/server.py: Flask control surface the CLI's container talks to (/deploy, /clean, /status), replacing the ad-hoc entrypoint. - ventis/cli.py: fold `build` into `deploy`, support the .car artifact layout (.car/app sources, .car/config declarations, .car/stubs), and resolve env_file against the project dir so it matches how the GlobalController resolves it at runtime. - GlobalController: persist a dashed-uuid project_id and publish the controller identity to Redis. - OTLP exporter: generate Future.id at 64 bits (secrets.token_hex(8)) so it is a valid OTel span_id without truncation, and cost lookups that fail (no pricing table on a local deploy) now cost at 0 instead of dropping the whole telemetry row. - stub_generator: a stub is written to exactly one location, the path of the entrypoint it replaces, rather than being duplicated at the flat basename as well. Flat is only the fallback for a stub with no entrypoint mapping or one whose mapping escapes the build context. Carries over the placement half of 692d17c from feature/all-the-files, which never reached this line; the entrypoint-adjacent YAML discovery from that commit is deliberately left out, since the .car layout already resolves declarations from .car/config. - stub_generator: fail loudly when an agent has no declaration or entrypoint. Co-Authored-By: Claude Opus 5 (1M context) --- tests/run_tests.sh | 9 +- tests/test_cli.py | 113 +++++++++++-- tests/test_deploy.py | 2 +- tests/test_error_propagation.py | 2 +- tests/test_future.py | 4 +- tests/test_global_controller_identity.py | 8 +- tests/test_global_controller_project_id.py | 70 ++++++++ tests/test_otel_exporter_fields.py | 4 +- tests/test_stub_generator.py | 53 +++++++ tests/test_ventis_context.py | 2 +- ventis/Dockerfile | 20 +++ ventis/OTLP_Exporter/convert.py | 9 +- ventis/OTLP_Exporter/db.py | 35 ++-- ventis/OTLP_Exporter/otel_exporter.py | 7 +- ventis/README.md | 8 + ventis/cli.py | 150 +++++++++++------- ventis/{llm => controller}/bedrock.py | 4 +- .../cloud_provider_logic/EC2/_runtime.py | 2 +- .../cloud_provider_logic/Local/_runtime.py | 117 ++++++++------ ventis/{ => controller}/deploy.py | 4 +- ventis/{ => controller}/future.py | 15 +- ventis/controller/global_controller.py | 52 ++++-- ventis/controller/local_controller.py | 6 +- .../controller/local_controller_frontend.py | 4 +- ventis/{ => controller}/utils/grpc_options.py | 0 ventis/{ => controller}/utils/redis_client.py | 0 ventis/controller/utils/telemetry_logging.py | 2 +- ventis/{ => controller}/ventis_context.py | 0 ventis/llm/__init__.py | 0 ventis/server.py | 70 ++++++++ ventis/stub_generator.py | 51 +++--- ventis/utils/__init__.py | 1 - 32 files changed, 612 insertions(+), 212 deletions(-) create mode 100644 tests/test_global_controller_project_id.py create mode 100644 ventis/Dockerfile create mode 100644 ventis/README.md rename ventis/{llm => controller}/bedrock.py (94%) rename ventis/{ => controller}/deploy.py (98%) rename ventis/{ => controller}/future.py (94%) rename ventis/{ => controller}/utils/grpc_options.py (100%) rename ventis/{ => controller}/utils/redis_client.py (100%) rename ventis/{ => controller}/ventis_context.py (100%) delete mode 100644 ventis/llm/__init__.py create mode 100644 ventis/server.py delete mode 100644 ventis/utils/__init__.py diff --git a/tests/run_tests.sh b/tests/run_tests.sh index e9f8386..c5556ec 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -30,13 +30,10 @@ cd "$TEST_DIR" echo ">> 1. Generating new project..." ventis new-project $PROJECT_NAME cd $PROJECT_NAME -grep -v 'gpu:' config/global_controller.yaml > config/global_controller.yaml.tmp -mv config/global_controller.yaml.tmp config/global_controller.yaml +grep -v 'gpu:' .car/config/global_controller.yaml > .car/config/global_controller.yaml.tmp +mv .car/config/global_controller.yaml.tmp .car/config/global_controller.yaml -echo ">> 2. Building agents (ventis build)..." -ventis build - -echo ">> 3. Deploying workflow (ventis deploy)..." +echo ">> 2. Building and deploying workflow (ventis deploy)..." ventis deploy & DEPLOY_PID=$! diff --git a/tests/test_cli.py b/tests/test_cli.py index 406b95d..44b9270 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,12 +23,14 @@ def _fake_controller_module(self, controller): @patch("atexit.register") @patch("signal.signal") + @patch("ventis.cli._run_build") @patch("ventis.cli._ensure_grpc_stubs_importable") @patch("ventis.cli._preflight_ec2_deploy") def test_deploy_skips_ec2_preflight_for_local_config( self, preflight, ensure_grpc, + _run_build, _signal_patch, _atexit_patch, ): @@ -54,12 +56,14 @@ def test_deploy_skips_ec2_preflight_for_local_config( @patch("atexit.register") @patch("signal.signal") + @patch("ventis.cli._run_build") @patch("ventis.cli._ensure_grpc_stubs_importable") @patch("ventis.cli._preflight_ec2_deploy") def test_deploy_runs_ec2_preflight_for_ec2_config( self, preflight, ensure_grpc, + _run_build, _signal_patch, _atexit_patch, ): @@ -81,6 +85,36 @@ def test_deploy_runs_ec2_preflight_for_ec2_config( preflight.assert_called_once_with(config, os.getcwd()) controller.run.assert_called_once_with() + @patch("atexit.register") + @patch("signal.signal") + @patch("ventis.cli._run_build") + @patch("ventis.cli._ensure_grpc_stubs_importable") + @patch("ventis.cli._preflight_ec2_deploy") + def test_deploy_uses_car_when_present( + self, preflight, ensure_grpc, _run_build, _signal_patch, _atexit_patch + ): + controller = MagicMock() + controller_module = self._fake_controller_module(controller) + args = SimpleNamespace(config=".car/config/global_controller.yaml") + + with tempfile.TemporaryDirectory() as tmpdir, patch( + "ventis.cli.os.path.isfile", return_value=True + ), patch( + "ventis.cli._load_config", return_value={"agents": []} + ), patch.dict( + sys.modules, {"ventis.controller.global_controller": controller_module} + ): + Path(tmpdir, ".car").mkdir() + cwd = os.getcwd() + os.chdir(tmpdir) + try: + cli.cmd_deploy(args) + finally: + os.chdir(cwd) + + ensure_grpc.assert_called_once_with(os.path.join(os.path.realpath(tmpdir), ".car")) + preflight.assert_not_called() + @patch("ventis.cli._ensure_grpc_stubs_importable") @patch("ventis.cli._require_docker_for_ec2") def test_preflight_does_not_require_ssh_fields(self, require_docker, ensure_grpc): @@ -103,28 +137,43 @@ class CliBuildTests(unittest.TestCase): def _run_build( self, project_dir, agent_yaml_paths, buildx_available, platform="linux/amd64" ): - """Run cmd_build against project_dir with docker/subprocess calls mocked. + """Run _run_build against project_dir with docker/subprocess calls mocked. Returns (docker_calls, generate_docker_mock, generate_workflow_docker_mock). """ - config_path = project_dir / "config" / "global_controller.yaml" - args = SimpleNamespace(config=str(config_path)) + artifact_root = ( + project_dir / ".car" if (project_dir / ".car").is_dir() else project_dir + ) + config_path = artifact_root / "config" / "global_controller.yaml" docker_calls = [] def fake_run(cmd, check): docker_calls.append(cmd) return SimpleNamespace(returncode=0) + def fake_glob(pattern): + if pattern.endswith("*.proto"): + return ["proto/a.proto"] + if agent_yaml_paths: + self.assertEqual( + os.path.realpath(Path(pattern).parent), + os.path.realpath(Path(agent_yaml_paths[0]).parent), + ) + return agent_yaml_paths + + def fake_generate_stub(yaml_path, _output_path): + with open(yaml_path) as f: + self.assertIn("agent", yaml.safe_load(f)) + with ( patch( "ventis.cli._get_package_dir", return_value=str(project_dir / "package"), ), + patch("ventis.cli.glob.glob", side_effect=fake_glob), patch( - "ventis.cli.glob.glob", - side_effect=[agent_yaml_paths, ["proto/a.proto"]], + "ventis.stub_generator.generate_stub", side_effect=fake_generate_stub ), - patch("ventis.stub_generator.generate_stub"), patch("ventis.stub_generator.generate_docker") as generate_docker, patch( "ventis.stub_generator.generate_workflow_docker" @@ -136,7 +185,7 @@ def fake_run(cmd, check): cwd = os.getcwd() os.chdir(project_dir) try: - cli.cmd_build(args) + cli._run_build(str(config_path)) finally: os.chdir(cwd) @@ -240,6 +289,32 @@ def test_build_uses_buildx_bake_when_available(self): ) self.assertEqual(targets["workflow"]["tags"], ["ventis-workflow"]) + def test_build_uses_car_when_present(self): + with tempfile.TemporaryDirectory() as tmpdir: + project_dir = Path(tmpdir) + artifact_root = project_dir / ".car" + source_root = artifact_root / "app" + source_root.mkdir(parents=True) + source_yaml = self._write_agent_and_workflow_config(source_root) + source_root.joinpath("config").rename(artifact_root / "config") + agent_yaml = artifact_root / "config" / source_yaml.name + source_yaml.rename(agent_yaml) + + manifest = artifact_root / "config" / "global_controller.yaml" + _, generate_docker, generate_workflow_docker = self._run_build( + project_dir, [str(manifest), str(agent_yaml)], buildx_available=True + ) + + for call in (generate_docker, generate_workflow_docker): + self.assertEqual( + os.path.realpath(call.call_args.kwargs["project_dir"]), + os.path.realpath(source_root), + ) + self.assertEqual( + os.path.realpath(generate_docker.call_args.kwargs["output_dir"]), + os.path.realpath(artifact_root / "docker_container" / "ExampleAgent"), + ) + def test_build_with_no_agents_builds_nothing(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) @@ -252,7 +327,7 @@ def test_build_with_no_agents_builds_nothing(self): self.assertFalse(any(call[0] == "docker" for call in docker_calls)) - def test_build_skips_agent_without_entrypoint(self): + def test_build_fails_when_stub_cannot_be_generated(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) (project_dir / "config").mkdir() @@ -268,9 +343,8 @@ def test_build_skips_agent_without_entrypoint(self): ) ) - docker_calls, _, _ = self._run_build(project_dir, [], buildx_available=True) - - self.assertFalse(any(call[0] == "docker" for call in docker_calls)) + with self.assertRaises(SystemExit): + self._run_build(project_dir, [], buildx_available=True) def _write_requirements_config(self, project_dir): """Scaffold one plain agent, one agent with `requirements`, one workflow with `requirements`.""" @@ -373,5 +447,22 @@ def test_build_ignores_non_list_requirements(self): self.assertEqual(generate_docker.call_args.kwargs["requirements"], []) +class CliCleanTests(unittest.TestCase): + def test_clean_uses_car_when_present(self): + with tempfile.TemporaryDirectory() as tmpdir: + project_dir = Path(tmpdir) + (project_dir / ".car" / "stubs").mkdir(parents=True) + (project_dir / "stubs").mkdir() + cwd = os.getcwd() + os.chdir(project_dir) + try: + cli.cmd_clean(SimpleNamespace()) + finally: + os.chdir(cwd) + + self.assertFalse((project_dir / ".car" / "stubs").exists()) + self.assertTrue((project_dir / "stubs").exists()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 3f029cb..3c02008 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -import ventis.deploy as deploy_module +import ventis.controller.deploy as deploy_module class _FakeRedis: diff --git a/tests/test_error_propagation.py b/tests/test_error_propagation.py index 82da262..59c6eba 100644 --- a/tests/test_error_propagation.py +++ b/tests/test_error_propagation.py @@ -17,7 +17,7 @@ from ventis.controller.local_controller import LocalController from ventis.controller.local_controller_frontend import LocalControllerServicer -from ventis.future import Future +from ventis.controller.future import Future import local_controler_pb2 diff --git a/tests/test_future.py b/tests/test_future.py index e190426..4914b29 100644 --- a/tests/test_future.py +++ b/tests/test_future.py @@ -13,8 +13,8 @@ ), ) -import ventis.future as future_module -import ventis.ventis_context as ventis_context +import ventis.controller.future as future_module +import ventis.controller.ventis_context as ventis_context class _FakeRedis: diff --git a/tests/test_global_controller_identity.py b/tests/test_global_controller_identity.py index e4337f7..76a6689 100644 --- a/tests/test_global_controller_identity.py +++ b/tests/test_global_controller_identity.py @@ -103,14 +103,16 @@ def test_a_second_call_with_a_new_config_overwrites_the_published_value(self): }, ) - def test_missing_project_id_or_database_publishes_safe_defaults(self): - controller = _bare_controller({}) + def test_missing_database_publishes_safe_default(self): + # project_id is always populated by _load_config() by the time _write_identity() + # runs -- only database_url has a real "unset" case to default here. + controller = _bare_controller({"project_id": "11111111-1111-1111-1111-111111111111"}) controller._write_identity() self.assertEqual( controller.redis.hgetall(GlobalController.IDENTITY_KEY), - {"project_id": "0", "database_url": ""}, + {"project_id": "11111111-1111-1111-1111-111111111111", "database_url": ""}, ) diff --git a/tests/test_global_controller_project_id.py b/tests/test_global_controller_project_id.py new file mode 100644 index 0000000..1feebc8 --- /dev/null +++ b/tests/test_global_controller_project_id.py @@ -0,0 +1,70 @@ +"""_load_config() must mint a project_id when a config file omits one, and persist it back +to the file so the same value survives a reload_config() or process restart -- not a fresh +uuid on every load. +""" + +import os +import re +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +import yaml + +from ventis.controller.global_controller import GlobalController + +UUID_HEX_RE = re.compile(r"^[0-9a-f]{32}$") + + +def _write_config(body): + f = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) + f.write(body) + f.close() + return f.name + + +class LoadConfigProjectIdTests(unittest.TestCase): + def test_generates_and_persists_project_id_when_missing(self): + config_path = _write_config("agents: []\npoll_interval: 5\n") + try: + config = GlobalController._load_config(config_path) + + self.assertTrue(UUID_HEX_RE.match(config["project_id"])) + + with open(config_path) as f: + on_disk = yaml.safe_load(f) + self.assertEqual(on_disk["project_id"], config["project_id"]) + finally: + os.unlink(config_path) + + def test_reload_reuses_the_persisted_project_id_instead_of_minting_a_new_one(self): + config_path = _write_config("agents: []\npoll_interval: 5\n") + try: + first = GlobalController._load_config(config_path) + second = GlobalController._load_config(config_path) + + self.assertEqual(first["project_id"], second["project_id"]) + finally: + os.unlink(config_path) + + def test_existing_project_id_is_left_untouched(self): + config_path = _write_config( + 'agents: []\nproject_id: "11111111-1111-1111-1111-111111111111"\n' + ) + try: + config = GlobalController._load_config(config_path) + + self.assertEqual(config["project_id"], "11111111-1111-1111-1111-111111111111") + + with open(config_path) as f: + contents = f.read() + # No second project_id line got appended alongside the existing one. + self.assertEqual(contents.count("project_id"), 1) + finally: + os.unlink(config_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index b74177d..f6c2074 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -29,7 +29,7 @@ def test_init_db_creates_waiting_table_with_full_schema(self): def test_fields_are_normalized_and_added_to_span(self): db.init_db(self.db_path) raw = { - "future_id": "00112233445566778899aabbccddeeff", + "future_id": "0011223344556677", # 64-bit (16 hex chars), matches Future.id's format "request_id": "ffeeddccbbaa99887766554433221100", "service": "PriceAgent", "method": "get_history", @@ -61,7 +61,7 @@ def test_fields_are_normalized_and_added_to_span(self): def test_error_message_is_wired_from_redis_error_field(self): db.init_db(self.db_path) raw = { - "future_id": "11112222333344445555666677778888", + "future_id": "1111222233334444", # 64-bit (16 hex chars), matches Future.id's format "request_id": "88887777666655554444333322221111", "service": "AdvisorAgent", "method": "summarize", diff --git a/tests/test_stub_generator.py b/tests/test_stub_generator.py index fb01f2e..916bf80 100644 --- a/tests/test_stub_generator.py +++ b/tests/test_stub_generator.py @@ -11,6 +11,7 @@ from ventis.stub_generator import ( BASE_AGENT_REQUIREMENTS, BASE_WORKFLOW_REQUIREMENTS, + _stub_destination, generate_docker, generate_workflow_docker, ) @@ -86,5 +87,57 @@ def test_per_workflow_requirements_are_appended_to_base(self): self.assertEqual(requirements, BASE_WORKFLOW_REQUIREMENTS + ["yfinance"]) +class StubDestinationTests(unittest.TestCase): + """A stub replaces the real module at its entrypoint path, so it is written + to exactly that one location. Flat is only a fallback for a stub with no + entrypoint mapping, or one whose mapping escapes the build context. + """ + + def test_unmapped_stub_falls_back_to_flat(self): + self.assertEqual(_stub_destination("/stubs/split_agent.py", {}), "split_agent.py") + + def test_entrypoint_mapping_is_the_only_destination(self): + destination = _stub_destination( + "/stubs/split_agent.py", {"split_agent.py": "agents/split_agent.py"} + ) + self.assertEqual(destination, "agents/split_agent.py") + + def test_flat_entrypoint_stays_flat(self): + destination = _stub_destination( + "/stubs/split_agent.py", {"split_agent.py": "split_agent.py"} + ) + self.assertEqual(destination, "split_agent.py") + + def test_unsafe_entrypoint_falls_back_to_flat(self): + destination = _stub_destination( + "/stubs/split_agent.py", {"split_agent.py": "../../etc/passwd"} + ) + self.assertEqual(destination, "split_agent.py") + + +class GenerateWorkflowDockerStubPlacementTests(unittest.TestCase): + def test_stub_lands_only_at_its_entrypoint_path(self): + with tempfile.TemporaryDirectory() as tmpdir: + workflow_file = Path(tmpdir) / "workflow.py" + workflow_file.write_text("from agents.split_agent import SplitAgent\n") + + stub_file = Path(tmpdir) / "stubs" / "split_agent.py" + stub_file.parent.mkdir() + stub_file.write_text("class SplitAgent:\n pass\n") + + output_dir = os.path.join(tmpdir, "out") + generate_workflow_docker( + str(workflow_file), + [str(stub_file)], + output_dir=output_dir, + stub_entrypoints={"split_agent.py": "agents/split_agent.py"}, + ) + + nested_path = Path(output_dir) / "agents" / "split_agent.py" + flat_path = Path(output_dir) / "split_agent.py" + self.assertIn("class SplitAgent", nested_path.read_text()) + self.assertFalse(flat_path.exists(), "stub must not be duplicated flat") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ventis_context.py b/tests/test_ventis_context.py index bec4122..f860d1f 100644 --- a/tests/test_ventis_context.py +++ b/tests/test_ventis_context.py @@ -4,7 +4,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -import ventis.ventis_context as ventis_context +import ventis.controller.ventis_context as ventis_context class VentisContextTests(unittest.TestCase): diff --git a/ventis/Dockerfile b/ventis/Dockerfile new file mode 100644 index 0000000..1814548 --- /dev/null +++ b/ventis/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y docker.io && rm -rf /var/lib/apt/lists/* + +COPY . /ventis +RUN pip install /ventis + +# global_controller.py bare-imports these; pip install only ships the .proto source. +RUN python -m grpc_tools.protoc \ + -I/ventis/ventis/controller/proto \ + --python_out=/usr/local/lib/python3.11/site-packages \ + --grpc_python_out=/usr/local/lib/python3.11/site-packages \ + /ventis/ventis/controller/proto/local_controler.proto + +EXPOSE 8000 + +ENTRYPOINT ["python", "-m", "ventis.server"] + + +# to run: docker build -f ventis/Dockerfile -t saakeths/canyonos:latest . diff --git a/ventis/OTLP_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py index 5e6ac74..72e2342 100644 --- a/ventis/OTLP_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -29,11 +29,12 @@ def waiting_row_to_span(row): row = dict(row) trace_id = int(row["session_id"], 16) - span_id = int.from_bytes(bytes.fromhex(row["future_id"])[:8], "big") + # future_id/parent_id are already 64-bit (Future.id is generated at that + # width directly -- see ventis/controller/future.py), matching OTel's + # span_id, so no truncation is needed here. + span_id = int(row["future_id"], 16) parent_id = row.get("parent_id") - parent_span_id = ( - int.from_bytes(bytes.fromhex(parent_id)[:8], "big") if parent_id else None - ) + parent_span_id = int(parent_id, 16) if parent_id else None context = SpanContext( trace_id=trace_id, span_id=span_id, is_remote=False, trace_flags=_SAMPLED diff --git a/ventis/OTLP_Exporter/db.py b/ventis/OTLP_Exporter/db.py index 005ba71..f1438f7 100644 --- a/ventis/OTLP_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -134,21 +134,30 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH # Cost figures are only meaningful once the future has finished, so skip # computing them until then rather than recomputing on every poll. if finished_at is not None: - token_cost = ( - pricing.compute_token_cost( - raw.get("model"), input_token_count, output_token_count + # Cost lookups can fail independently of the telemetry itself (e.g. + # no aws_instance_pricing table on a local-provider deployment) -- + # don't let that drop the whole row, just cost it at 0. + try: + token_cost = ( + pricing.compute_token_cost( + raw.get("model"), input_token_count, output_token_count + ) + * _TOKEN_COST_MULTIPLIER ) - * _TOKEN_COST_MULTIPLIER - ) - server_cost = ( - pricing.compute_server_cost( - redis_client.get(f"agent:{agent_id}:instance_type") - if redis_client is not None and agent_id - else None, - finished_at - started_at, + except Exception: + token_cost = 0.0 + try: + server_cost = ( + pricing.compute_server_cost( + redis_client.get(f"agent:{agent_id}:instance_type") + if redis_client is not None and agent_id + else None, + finished_at - started_at, + ) + * _SERVER_COST_MULTIPLIER ) - * _SERVER_COST_MULTIPLIER - ) + except Exception: + server_cost = 0.0 else: token_cost = 0.0 server_cost = 0.0 diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index 7e365b0..1ed210a 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -20,7 +20,7 @@ import time sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from ventis.utils.redis_client import RedisClient +from ventis.controller.utils.redis_client import RedisClient from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter as GrpcOTLPSpanExporter, @@ -243,7 +243,10 @@ def main(): signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _redis = RedisClient() # localhost:6379, db 0 -- same host as GlobalController + # GC reaches its own Redis via host.docker.internal (a sibling container, + # not the same network namespace, since GC runs on bridge networking) -- + # match that instead of plain localhost. + _redis = RedisClient(host="host.docker.internal") _last_destinations_raw = _redis.get(DESTINATIONS_KEY) _processors = _build_processors(_last_destinations_raw) logger.info("OTel exporter process started with %d destination(s).", len(_processors)) diff --git a/ventis/README.md b/ventis/README.md new file mode 100644 index 0000000..6b5675a --- /dev/null +++ b/ventis/README.md @@ -0,0 +1,8 @@ +# CanyonOS Platform + +Every folder in here is a separate process to be run. + +- controller: The control plane and manager +- OTLP_Exporter: The OTel Data Exporter +- server.py: Flask server that CLI connects to +- (soon) Instance_Manager: Responsible for scaling (currently in controller) \ No newline at end of file diff --git a/ventis/cli.py b/ventis/cli.py index c5a2b68..c66a106 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -1,10 +1,10 @@ """ Ventis CLI -Entry point for the `ventis` command. Provides three subcommands: +Entry point for the `ventis` command. Provides these subcommands: ventis new-project — Scaffold a new Ventis project - ventis build — Generate stubs and build Docker images - ventis deploy — Launch agents via the Global Controller + ventis deploy — Build (stubs + Docker images) then launch + agents via the Global Controller """ import argparse @@ -21,7 +21,8 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" -DEFAULT_CONFIG_PATH = "config/global_controller.yaml" +ARTIFACT_DIR_NAME = ".car" +SOURCE_DIR_NAME = "app" EC2_REQUIRED_CONFIG_KEYS = ( "ami_id", "subnet_id", @@ -53,6 +54,10 @@ def _load_config(config_path): return yaml.safe_load(f) +def _artifact_prefix(root): + return ARTIFACT_DIR_NAME if os.path.isdir(os.path.join(root, ARTIFACT_DIR_NAME)) else "" + + def _normalize_requirements(agent_cfg): """Return an agent's `requirements` list, or [] if absent/null/malformed.""" requirements = agent_cfg.get("requirements") or [] @@ -175,17 +180,35 @@ def cmd_new_project(args): logger.error("Templates directory not found at %s", templates_dir) sys.exit(1) - # Copy the entire templates tree into the new project - shutil.copytree(templates_dir, project_dir) + # Copy the entire templates tree into .car/app, then pull config and agent + # declarations up into .car/config, keeping generated artifacts (stubs, + # grpc_stubs, docker_container) siblings of the source under .car/. + artifact_root = os.path.join(project_dir, ARTIFACT_DIR_NAME) + source_root = os.path.join(artifact_root, SOURCE_DIR_NAME) + shutil.copytree(templates_dir, source_root) + + source_config = os.path.join(source_root, "config") + artifact_config = os.path.join(artifact_root, "config") + if os.path.isdir(source_config): + shutil.move(source_config, artifact_root) + else: + os.makedirs(artifact_config) + + source_agents = os.path.join(source_root, "agents") + for declaration in glob.glob(os.path.join(source_agents, "*.yaml")): + shutil.move(declaration, artifact_config) + + readme = os.path.join(source_root, "README.md") + if os.path.isfile(readme): + shutil.move(readme, project_dir) # Create empty output directories - os.makedirs(os.path.join(project_dir, "stubs"), exist_ok=True) - os.makedirs(os.path.join(project_dir, "grpc_stubs"), exist_ok=True) + os.makedirs(os.path.join(artifact_root, "stubs"), exist_ok=True) + os.makedirs(os.path.join(artifact_root, "grpc_stubs"), exist_ok=True) logger.info("Created new Ventis project: %s", project_dir) logger.info("") logger.info(" cd %s", project_name) - logger.info(" ventis build") logger.info(" ventis deploy") @@ -194,28 +217,31 @@ def cmd_new_project(args): # ------------------------------------------------------------------ # -def cmd_build(args): +def _run_build(config_path): """ Generate stubs, compile gRPC protos, generate Docker contexts, and build Docker images. - Must be run from the project root (where config/ lives). + Must be run from the project root (where config/ lives). Invoked as the + first phase of `ventis deploy`. """ - config_path = args.config if not os.path.isfile(config_path): logger.error("Config file not found: %s", config_path) sys.exit(1) config = _load_config(config_path) agents = config.get("agents", []) - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir + source_root = os.path.join(artifact_root, SOURCE_DIR_NAME) if prefix else project_dir package_dir = _get_package_dir() # -------------------------------------------------------------- # # Step 1: Discover agent YAML files and generate Python stubs # # -------------------------------------------------------------- # - agents_dir = os.path.join(project_dir, "agents") - stubs_dir = os.path.join(project_dir, "stubs") + declarations_dir = os.path.join(artifact_root, "config" if prefix else "agents") + stubs_dir = os.path.join(artifact_root, "stubs") os.makedirs(stubs_dir, exist_ok=True) from ventis.stub_generator import ( @@ -224,9 +250,9 @@ def cmd_build(args): generate_workflow_docker, ) - yaml_files = glob.glob(os.path.join(agents_dir, "*.yaml")) + yaml_files = glob.glob(os.path.join(declarations_dir, "*.yaml")) if not yaml_files: - logger.warning("No agent YAML files found in %s", agents_dir) + logger.warning("No agent YAML files found in %s", declarations_dir) import yaml @@ -238,9 +264,19 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path - # Maps each generated stub's basename to its agent's entrypoint path, so a - # stub can also be placed at its nested, entrypoint-mirrored location. + # Maps each generated stub's basename to its agent's entrypoint path, which + # is the single location the stub is written to and copied to. entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + missing_stubs = [ + a["name"] + for a in agents + if a.get("type", "agent") != "workflow" + and (a["name"] not in yaml_by_name or not a.get("entrypoint")) + ] + if missing_stubs: + logger.error("Cannot generate stubs for agents: %s", ", ".join(missing_stubs)) + sys.exit(1) + stub_entrypoints = { f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] for n, p in yaml_by_name.items() @@ -248,9 +284,12 @@ def cmd_build(args): } stub_paths = [] - for yaml_path in yaml_files: - base_name = os.path.splitext(os.path.basename(yaml_path))[0] - output_path = os.path.join(stubs_dir, f"{base_name}.py") + for agent_name, yaml_path in yaml_by_name.items(): + entrypoint = entrypoints_by_name.get(agent_name) + if not entrypoint: + continue + output_path = os.path.join(stubs_dir, entrypoint) + os.makedirs(os.path.dirname(output_path), exist_ok=True) logger.info("Generating stub: %s -> %s", yaml_path, output_path) generate_stub(yaml_path, output_path) stub_paths.append(output_path) @@ -258,7 +297,7 @@ def cmd_build(args): # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # - grpc_stubs_dir = os.path.join(project_dir, "grpc_stubs") + grpc_stubs_dir = os.path.join(artifact_root, "grpc_stubs") os.makedirs(grpc_stubs_dir, exist_ok=True) proto_dir = os.path.join(package_dir, "controller", "proto") @@ -296,12 +335,12 @@ def cmd_build(args): ) continue - workflow_path = os.path.join(project_dir, workflow_file) + workflow_path = os.path.join(source_root, workflow_file) if not os.path.isfile(workflow_path): logger.error("Workflow file not found: %s", workflow_path) continue - docker_context = os.path.join(project_dir, "docker_container", "Workflow") + docker_context = os.path.join(artifact_root, "docker_container", "Workflow") logger.info("Generating workflow Docker context for '%s'", agent_name) generate_workflow_docker( workflow_path, @@ -309,10 +348,8 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), - project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + project_dir=source_root, requirements=_normalize_requirements(agent_cfg), - project_dir=project_dir, # Stubs are placed both flat and at their entrypoint-mirrored path, # so both flat and nested import styles resolve to the stub. stub_entrypoints=stub_entrypoints, @@ -327,7 +364,7 @@ def cmd_build(args): ) continue - agent_file = os.path.join(project_dir, entrypoint) + agent_file = os.path.join(source_root, entrypoint) if not os.path.isfile(agent_file): logger.error("Agent file not found: %s", agent_file) continue @@ -341,7 +378,7 @@ def cmd_build(args): ) continue - docker_context = os.path.join(project_dir, "docker_container", agent_name) + docker_context = os.path.join(artifact_root, "docker_container", agent_name) logger.info("Generating Docker context for '%s'", agent_name) generate_docker( matching_yaml, @@ -349,10 +386,8 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, - project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + project_dir=source_root, requirements=_normalize_requirements(agent_cfg), - project_dir=project_dir, # Same reasoning as the workflow call above: stubs are placed both # flat and at their entrypoint-mirrored path. stub_entrypoints=stub_entrypoints, @@ -372,7 +407,7 @@ def cmd_build(args): if not bake_targets: logger.info("No Docker images to build.") elif _docker_available() and _docker_available(("docker", "buildx", "version")): - docker_container_dir = os.path.join(project_dir, "docker_container") + docker_container_dir = os.path.join(artifact_root, "docker_container") os.makedirs(docker_container_dir, exist_ok=True) bake_file_path = os.path.join(docker_container_dir, "docker-bake.json") _write_bake_file(bake_targets, bake_file_path, _docker_platform()) @@ -418,24 +453,31 @@ def cmd_deploy(args): logger.error("Config file not found: %s", config_path) sys.exit(1) + # Build first (stubs, protos, Docker contexts, images), then deploy them. + # `ventis build` was merged into `ventis deploy`. + _run_build(config_path) + config = _load_config(config_path) - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir # Fail here rather than after a fleet of containers is already up without - # the API keys they need. + # the API keys they need. base_dir matches GlobalController, which resolves + # env_file against its cwd -- any other base rejects a file it would find. try: resolve_env_file(config, base_dir=project_dir) except ValueError as e: logger.error("%s", e) sys.exit(1) - _ensure_grpc_stubs_importable(project_dir) + _ensure_grpc_stubs_importable(artifact_root) if any( agent.get("provider", "local").upper() == "EC2" for agent in config.get("agents", []) ): - _preflight_ec2_deploy(config, project_dir) + _preflight_ec2_deploy(config, artifact_root) from ventis.controller.global_controller import GlobalController @@ -476,12 +518,14 @@ def cmd_clean(args): """ Remove generated stubs, gRPC files, and Docker build contexts. """ - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir paths_to_clean = [ - os.path.join(project_dir, "stubs"), - os.path.join(project_dir, "grpc_stubs"), - os.path.join(project_dir, "docker_container"), + os.path.join(artifact_root, "stubs"), + os.path.join(artifact_root, "grpc_stubs"), + os.path.join(artifact_root, "docker_container"), ] for path in paths_to_clean: @@ -503,6 +547,9 @@ def cmd_clean(args): def main(): + default_config_path = os.path.join( + _artifact_prefix(os.getcwd()), "config", "global_controller.yaml" + ) parser = argparse.ArgumentParser( prog="ventis", description="Ventis — Distributed Agent Orchestration Framework", @@ -517,29 +564,16 @@ def main(): new_proj.add_argument("name", help="Name of the project directory to create") new_proj.set_defaults(func=cmd_new_project) - # ventis build - build = subparsers.add_parser( - "build", - help="Generate stubs, compile protos, and build Docker images", - ) - build.add_argument( - "-c", - "--config", - default=DEFAULT_CONFIG_PATH, - help=f"Path to global controller config (default: {DEFAULT_CONFIG_PATH})", - ) - build.set_defaults(func=cmd_build) - # ventis deploy deploy = subparsers.add_parser( "deploy", - help="Launch agents via the Global Controller", + help="Build stubs/images, then launch agents via the Global Controller", ) deploy.add_argument( "-c", "--config", - default=DEFAULT_CONFIG_PATH, - help=f"Path to global controller config (default: {DEFAULT_CONFIG_PATH})", + default=default_config_path, + help=f"Path to global controller config (default: {default_config_path})", ) deploy.set_defaults(func=cmd_deploy) diff --git a/ventis/llm/bedrock.py b/ventis/controller/bedrock.py similarity index 94% rename from ventis/llm/bedrock.py rename to ventis/controller/bedrock.py index f350b69..97c6a3f 100644 --- a/ventis/llm/bedrock.py +++ b/ventis/controller/bedrock.py @@ -1,8 +1,8 @@ import os try: - from ventis.utils.redis_client import RedisClient - import ventis.ventis_context as ventis_context + from ventis.controller.utils.redis_client import RedisClient + import ventis.controller.ventis_context as ventis_context except ImportError: from redis_client import RedisClient import ventis_context diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 7e4e9ab..a1cbb4d 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -25,7 +25,7 @@ from ventis.controller.utils.env_file import env_file_args from ventis.controller.utils.redis_utils import _wait_for_redis -from ventis.utils.redis_client import RedisClient +from ventis.controller.utils.redis_client import RedisClient logger = logging.getLogger(__name__) diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index a387f7b..1a84e56 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -15,6 +15,7 @@ DEFAULT_HOST = "localhost" CONTAINER_PORT = 50051 PROVIDER = "local" +MAX_PORT_ATTEMPTS = 50 _controller = None @@ -62,9 +63,6 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): redis_host = provisioned["redis_host"] runtime_id = provisioned["runtime_id"] - endpoint = routing_endpoint_for(provisioned) - _require_controller().redis.set(f"controller:{endpoint}:agent_id", agent_id) - inspect = _require_controller()._run_cmd( ["docker", "inspect", "-f", "{{.State.Running}}", runtime_id], host, user ) @@ -76,53 +74,72 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): ) _require_controller()._run_cmd(["docker", "rm", "-f", runtime_id], host, user) - cmd = [ - "docker", - "run", - "-d", - "-it", - "--add-host=host.docker.internal:host-gateway", - "--name", - runtime_id, - "-p", - f"{host_port}:{CONTAINER_PORT}", - "-e", - f"VENTIS_AGENT_PORT={host_port}", - "-e", - f"VENTIS_AGENT_HOST={redis_host}", - "-e", - f"VENTIS_REDIS_HOST={redis_host}", - "-e", - f"VENTIS_REDIS_PORT={spec.get('redis_port', 6379)}", - "-e", - f"VENTIS_POLL_INTERVAL={_require_controller().config.get('poll_interval', 5)}", - ] - if ctrl_type == "workflow": - cmd.extend(["-p", f"{spec.get('api_port', 8080)}:8080"]) - config = _require_controller().config - db_url = config.get("database", {}).get("url") - project_id = config.get("project_id") - if db_url: - cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) - if project_id: - cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) - if resources.get("cpu"): - cmd.extend(["--cpus", str(resources["cpu"])]) - if resources.get("memory"): - cmd.extend(["--memory", f"{resources['memory']}m"]) - if resources.get("gpu"): - cmd.extend(["--gpus", str(resources["gpu"])]) - - # User secrets from `env_file`. Explicit -e flags above still win, so a - # stray VENTIS_* line in someone's .env cannot break agent wiring. - with env_file_args( - _require_controller(), host, user, runtime_id, _is_local_host(host) - ) as env_args: - cmd.extend(env_args) - cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) - if result.returncode != 0: - raise RuntimeError(f"Failed to launch {runtime_id}") + for attempt in range(MAX_PORT_ATTEMPTS): + cmd = [ + "docker", + "run", + "-d", + "-it", + "--add-host=host.docker.internal:host-gateway", + "--name", + runtime_id, + "-p", + f"{host_port}:{CONTAINER_PORT}", + "-e", + f"VENTIS_AGENT_PORT={host_port}", + "-e", + f"VENTIS_AGENT_HOST={redis_host}", + "-e", + f"VENTIS_REDIS_HOST={redis_host}", + "-e", + f"VENTIS_REDIS_PORT={spec.get('redis_port', 6379)}", + "-e", + f"VENTIS_POLL_INTERVAL={_require_controller().config.get('poll_interval', 5)}", + ] + if ctrl_type == "workflow": + cmd.extend(["-p", f"{spec.get('api_port', 8080)}:8080"]) + config = _require_controller().config + db_url = config.get("database", {}).get("url") + project_id = config.get("project_id") + if db_url: + cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) + if project_id: + cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) + if resources.get("cpu"): + cmd.extend(["--cpus", str(resources["cpu"])]) + if resources.get("memory"): + cmd.extend(["--memory", f"{resources['memory']}m"]) + if resources.get("gpu"): + cmd.extend(["--gpus", str(resources["gpu"])]) + + # User secrets from `env_file`. Explicit -e flags above still win, so a + # stray VENTIS_* line in someone's .env cannot break agent wiring. + with env_file_args( + _require_controller(), host, user, runtime_id, _is_local_host(host) + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _require_controller()._run_cmd(cmd, host, user) + + if result.returncode == 0: + break + if "port is already allocated" in (result.stderr or ""): + # `docker run` leaves a `Created`-but-never-started container behind + # under this name when the port bind fails. Remove it before + # retrying with a new port, or the retry hits a name conflict + # instead of the port conflict we're trying to work around. + _require_controller()._run_cmd(["docker", "rm", "-f", runtime_id], host, user) + host_port += 1 + continue + raise RuntimeError(f"Failed to launch {runtime_id}: {result.stderr}") + else: + raise RuntimeError( + f"Failed to launch {runtime_id}: no free port found after " + f"{MAX_PORT_ATTEMPTS} attempts" + ) + + endpoint = f"{_container_routing_host(host)}:{host_port}" + _require_controller().redis.set(f"controller:{endpoint}:agent_id", agent_id) instance = { "agent_name": agent_name, diff --git a/ventis/deploy.py b/ventis/controller/deploy.py similarity index 98% rename from ventis/deploy.py rename to ventis/controller/deploy.py index d197342..47de634 100644 --- a/ventis/deploy.py +++ b/ventis/controller/deploy.py @@ -17,7 +17,7 @@ def my_workflow(query: str): """ try: - import ventis.ventis_context as ventis_context + import ventis.controller.ventis_context as ventis_context except ImportError: import ventis_context import json @@ -33,7 +33,7 @@ def my_workflow(query: str): # Try to import from absolute package (local install) or fallback to flat file (Docker container) try: - from ventis.utils.redis_client import RedisClient + from ventis.controller.utils.redis_client import RedisClient except ImportError: from redis_client import RedisClient diff --git a/ventis/future.py b/ventis/controller/future.py similarity index 94% rename from ventis/future.py rename to ventis/controller/future.py index 68615f1..04b050a 100644 --- a/ventis/future.py +++ b/ventis/controller/future.py @@ -1,6 +1,6 @@ import time import json -import uuid +import secrets import sys import os import logging @@ -8,12 +8,12 @@ import grpc try: - import ventis.ventis_context as ventis_context + import ventis.controller.ventis_context as ventis_context except ImportError: import ventis_context try: - from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS + from ventis.controller.utils.grpc_options import GRPC_CHANNEL_OPTIONS except ImportError: from grpc_options import GRPC_CHANNEL_OPTIONS @@ -23,7 +23,7 @@ sys.path.insert(0, os.path.abspath("grpc_stubs")) try: - from ventis.utils.redis_client import RedisClient + from ventis.controller.utils.redis_client import RedisClient except ImportError: from redis_client import RedisClient import local_controler_pb2 @@ -63,8 +63,11 @@ def __init__(self, parent, service, method, args=None): args: arguments to be passed to the method """ - # initial value of future object - self.id = uuid.uuid4().hex + # initial value of future object. 64-bit (8 bytes / 16 hex chars) -- + # this doubles as the OTel span_id (convert.py), which is defined as + # 64-bit, so it's generated at that width directly instead of a + # 128-bit uuid4 that would need truncating later. + self.id = secrets.token_hex(8) # Grab the request_id from the thread-local context (set by deploy) self.request_id = ventis_context.get_request_id() diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index fc5dd38..c85dcbf 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -13,6 +13,7 @@ import sys import threading import time +import uuid from concurrent.futures import ThreadPoolExecutor import yaml @@ -28,11 +29,14 @@ send_runtime_information, send_agent_information, ) -from ventis.utils.redis_client import RedisClient -from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS - -# Add generated grpc_stubs from the local project to the path -sys.path.insert(0, os.path.abspath("grpc_stubs")) +from ventis.controller.utils.redis_client import RedisClient +from ventis.controller.utils.grpc_options import GRPC_CHANNEL_OPTIONS + +# Add generated grpc_stubs from the local project to the path. Projects using +# the .car artifact layout keep grpc_stubs under .car/; older/plain layouts +# keep it at the project root. +_artifact_prefix = ".car" if os.path.isdir(".car") else "" +sys.path.insert(0, os.path.abspath(os.path.join(_artifact_prefix, "grpc_stubs"))) import local_controler_pb2 import local_controler_pb2_grpc import grpc @@ -92,7 +96,7 @@ def __init__(self, config_path): self._last_metrics_poll_time = {} # (host, port) -> time.time() of last metrics read self._lc_stubs = {} # endpoint -> gRPC stub self.instance_manager = InstanceManager(self) - assign_project_id(self.config.get("project_id",0)) + assign_project_id(self.config.get("project_id")) # Clean up any stale containers from previous runs self._cleanup_stale_containers() @@ -188,12 +192,27 @@ def _cleanup_stale_containers(self): def _load_config(config_path): """Load the YAML config file after importing root .env values.""" project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) + # Under the .car layout, config lives at /.car/config, so the + # naive parent-of-parent lands on .car itself -- go up one more level + # to reach the actual project root where .env lives. + if os.path.basename(project_root) == ".car": + project_root = os.path.dirname(project_root) GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: config = yaml.safe_load(f) + if not config.get("project_id"): + config["project_id"] = GlobalController._assign_new_project_id(config_path) config = GlobalController._expand_env_value(config) return config + @staticmethod + def _assign_new_project_id(config_path): + """Generate a project_id and append it to the config file so it stays stable across reloads/restarts.""" + project_id = str(uuid.uuid4()) + with open(config_path, "a") as f: + f.write(f'project_id: "{project_id}"\n') + return project_id + @staticmethod def _load_dotenv(path): """Load simple KEY=VALUE entries without overriding existing environment values.""" @@ -261,7 +280,7 @@ def reload_config(self): self.env_file_path = resolve_env_file(self.config) self.controllers = self.config.get("agents", []) self.poll_interval = self.config.get("poll_interval", 5) - assign_project_id(self.config.get("project_id", 0)) + assign_project_id(self.config.get("project_id")) self._write_identity() self.instance_manager.publish_routing_snapshot(self.controllers) @@ -324,7 +343,7 @@ def _load_and_write_policies(self): def _write_identity(self): """Publish the current project/database identity to every node's Redis.""" payload = { - "project_id": str(self.config.get("project_id", 0)), + "project_id": str(self.config.get("project_id")), "database_url": self.config.get("database", {}).get("url") or "", } targets = list(self.node_redis.values()) or [self.redis] @@ -377,8 +396,11 @@ def _launch_redis_containers(self): redis_port = node_cfg["redis_port"] user = node_cfg["user"] container_name = f"ventis-redis-{host.replace('.', '-')}" - # For localhost, connect directly; for remote, connect via host IP - connect_host = "localhost" if host in ("localhost", "127.0.0.1") else host + # VENTIS_REDIS_HOST overrides the localhost case for a containerized GC; host/remote paths unchanged. + if host in ("localhost", "127.0.0.1"): + connect_host = os.environ.get("VENTIS_REDIS_HOST", "localhost") + else: + connect_host = host if self._redis_container_healthy(container_name, host, user, connect_host, redis_port): logger.info("Reusing existing Redis container %s on %s", container_name, host) @@ -562,7 +584,7 @@ def _poll_one_instance(self, instance): try: future_rows = pull_runtime_information(node_redis) self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) + future_rows, node_redis, self.config.get("project_id") ) # This is now legacy, keeping it for now, but will remove this later send_runtime_information( @@ -893,9 +915,9 @@ def stop(self): if __name__ == "__main__": - script_dir = os.path.dirname(os.path.abspath(__file__)) - project_root = os.path.join(script_dir, "..", "..") - default_config = os.path.join(project_root, "config", "global_controller.yaml") + default_config = os.path.join( + _artifact_prefix, "config", "global_controller.yaml" + ) import argparse @@ -904,7 +926,7 @@ def stop(self): "-c", "--config", default=default_config, - help="Path to the YAML config file (default: config/global_controller.yaml)", + help=f"Path to the YAML config file (default: {default_config})", ) args = parser.parse_args() diff --git a/ventis/controller/local_controller.py b/ventis/controller/local_controller.py index 8b5942e..8d9b525 100644 --- a/ventis/controller/local_controller.py +++ b/ventis/controller/local_controller.py @@ -18,8 +18,8 @@ try: from ventis.controller.local_controller_frontend import start_server from ventis.controller.utils.gpu_metrics import read_gpu_percent - from ventis.utils.redis_client import RedisClient - from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS + from ventis.controller.utils.redis_client import RedisClient + from ventis.controller.utils.grpc_options import GRPC_CHANNEL_OPTIONS except ImportError: from gpu_metrics import read_gpu_percent from local_controller_frontend import start_server @@ -32,7 +32,7 @@ sys.path.insert(0, os.path.abspath("grpc_stubs")) try: - import ventis.ventis_context as ventis_context + import ventis.controller.ventis_context as ventis_context except ImportError: import ventis_context import local_controler_pb2 diff --git a/ventis/controller/local_controller_frontend.py b/ventis/controller/local_controller_frontend.py index a5fcc25..722bd16 100644 --- a/ventis/controller/local_controller_frontend.py +++ b/ventis/controller/local_controller_frontend.py @@ -32,7 +32,7 @@ def __init__(self, my_endpoint="unknown"): redis_host = os.environ.get("VENTIS_REDIS_HOST", "localhost") redis_port = int(os.environ.get("VENTIS_REDIS_PORT", 6379)) try: - from ventis.utils.redis_client import RedisClient + from ventis.controller.utils.redis_client import RedisClient except ImportError: from redis_client import RedisClient self.redis = RedisClient(host=redis_host, port=redis_port) @@ -152,7 +152,7 @@ def _cleanup_request(self, request_id): def start_server(port=50051, my_endpoint="unknown"): """Start the gRPC server.""" try: - from ventis.utils.grpc_options import GRPC_SERVER_OPTIONS + from ventis.controller.utils.grpc_options import GRPC_SERVER_OPTIONS except ImportError: from grpc_options import GRPC_SERVER_OPTIONS diff --git a/ventis/utils/grpc_options.py b/ventis/controller/utils/grpc_options.py similarity index 100% rename from ventis/utils/grpc_options.py rename to ventis/controller/utils/grpc_options.py diff --git a/ventis/utils/redis_client.py b/ventis/controller/utils/redis_client.py similarity index 100% rename from ventis/utils/redis_client.py rename to ventis/controller/utils/redis_client.py diff --git a/ventis/controller/utils/telemetry_logging.py b/ventis/controller/utils/telemetry_logging.py index 7d911e2..503f3e1 100644 --- a/ventis/controller/utils/telemetry_logging.py +++ b/ventis/controller/utils/telemetry_logging.py @@ -7,7 +7,7 @@ from sqlalchemy import create_engine, text from ventis.controller.utils import pricing -from ventis.utils.redis_client import RedisClient +from ventis.controller.utils.redis_client import RedisClient logger = logging.getLogger(__name__) diff --git a/ventis/ventis_context.py b/ventis/controller/ventis_context.py similarity index 100% rename from ventis/ventis_context.py rename to ventis/controller/ventis_context.py diff --git a/ventis/llm/__init__.py b/ventis/llm/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ventis/server.py b/ventis/server.py new file mode 100644 index 0000000..8df8b0f --- /dev/null +++ b/ventis/server.py @@ -0,0 +1,70 @@ +import os +import signal +import subprocess +import sys + +from flask import Flask, jsonify, request + +app = Flask("ventis-server") + +# The project files are copied here (into a named volume) by `canyonos sync` / +# `canyonos deploy`. Deploy builds and launches against this path. +WORKSPACE_DIR = "/workspace" + +_gc_process = None + + +def _gc_running(): + return _gc_process is not None and _gc_process.poll() is None + + +@app.route("/new-project", methods=["POST"]) +def new_project(): + return jsonify({"error": "new-project runs locally via the CLI"}), 400 + + +@app.route("/deploy", methods=["POST"]) +def deploy(): + global _gc_process + + if _gc_running(): + return jsonify({"error": "already running"}), 409 + + data = request.get_json(force=True, silent=True) or {} + config_path = data.get("config_path", "config/global_controller.yaml") + full_path = os.path.join(WORKSPACE_DIR, config_path) + + if not os.path.isfile(full_path): + return jsonify({"error": f"config file not found: {full_path}"}), 400 + + # `ventis deploy` builds (stubs/protos/images) then launches the Global + # Controller. cwd is the workspace so build outputs land alongside the + # project files and the controller finds them. Build+deploy output streams + # to the container logs, which `canyonos deploy` tails. + _gc_process = subprocess.Popen( + [sys.executable, "-m", "ventis.cli", "deploy", "-c", config_path], + cwd=WORKSPACE_DIR, + ) + return jsonify({"status": "started", "pid": _gc_process.pid}), 200 + + +@app.route("/clean", methods=["POST"]) +def clean(): + global _gc_process + + if not _gc_running(): + return jsonify({"error": "not running"}), 409 + + _gc_process.send_signal(signal.SIGTERM) + _gc_process.wait() + _gc_process = None + return jsonify({"status": "stopped"}), 200 + + +@app.route("/status", methods=["GET"]) +def status(): + return jsonify({"running": _gc_running()}), 200 + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8000) diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 33824ff..1108be1 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -378,8 +378,8 @@ def generate_docker( # Copy general agent files files_to_copy += [ # (source_path, destination_filename) - (os.path.join(script_dir, "future.py"), "future.py"), - (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), + (os.path.join(script_dir, "controller", "future.py"), "future.py"), + (os.path.join(script_dir, "controller", "ventis_context.py"), "ventis_context.py"), ( os.path.join(script_dir, "controller", "local_controller.py"), "local_controller.py", @@ -388,26 +388,20 @@ def generate_docker( os.path.join(script_dir, "controller", "local_controller_frontend.py"), "local_controller_frontend.py", ), - (os.path.join(script_dir, "utils", "redis_client.py"), "redis_client.py"), - (os.path.join(script_dir, "utils", "grpc_options.py"), "grpc_options.py"), + (os.path.join(script_dir, "controller", "utils", "redis_client.py"), "redis_client.py"), + (os.path.join(script_dir, "controller", "utils", "grpc_options.py"), "grpc_options.py"), ( os.path.join(script_dir, "controller", "utils", "gpu_metrics.py"), "gpu_metrics.py", ), - (os.path.join(script_dir, "llm", "bedrock.py"), "bedrock.py"), + (os.path.join(script_dir, "controller", "bedrock.py"), "bedrock.py"), ] # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: - files_to_copy.append( - ( - os.path.abspath(stub_file), - _stub_destination(stub_file, stub_entrypoints or {}), - ) - ) - - files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) + destination = _stub_destination(stub_file, stub_entrypoints or {}) + files_to_copy.append((os.path.abspath(stub_file), destination)) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -417,6 +411,12 @@ def generate_docker( _copy_files(output_dir, files_to_copy) + # Copy the real agent entrypoint to the context root. + shutil.copy2( + os.path.abspath(agent_file), + os.path.join(output_dir, os.path.basename(agent_file)), + ) + # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -500,10 +500,9 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ - (os.path.abspath(workflow_file), workflow_basename), - (os.path.join(script_dir, "future.py"), "future.py"), - (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), - (os.path.join(script_dir, "deploy.py"), "deploy.py"), + (os.path.join(script_dir, "controller", "future.py"), "future.py"), + (os.path.join(script_dir, "controller", "ventis_context.py"), "ventis_context.py"), + (os.path.join(script_dir, "controller", "deploy.py"), "deploy.py"), ( os.path.join(script_dir, "controller", "local_controller.py"), "local_controller.py", @@ -512,8 +511,8 @@ def generate_workflow_docker( os.path.join(script_dir, "controller", "local_controller_frontend.py"), "local_controller_frontend.py", ), - (os.path.join(script_dir, "utils", "redis_client.py"), "redis_client.py"), - (os.path.join(script_dir, "utils", "grpc_options.py"), "grpc_options.py"), + (os.path.join(script_dir, "controller", "utils", "redis_client.py"), "redis_client.py"), + (os.path.join(script_dir, "controller", "utils", "grpc_options.py"), "grpc_options.py"), *[ (os.path.join(script_dir, "controller", "utils", name), name) for name in ("gpu_metrics.py", "session_logging.py") @@ -522,12 +521,8 @@ def generate_workflow_docker( # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - files_to_copy.append( - ( - os.path.abspath(stub_file), - _stub_destination(stub_file, stub_entrypoints or {}), - ) - ) + destination = _stub_destination(stub_file, stub_entrypoints or {}) + files_to_copy.append((os.path.abspath(stub_file), destination)) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -537,6 +532,12 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) + # Copy the real workflow entrypoint to the context root. + shutil.copy2( + os.path.abspath(workflow_file), + os.path.join(output_dir, workflow_basename), + ) + # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time diff --git a/ventis/utils/__init__.py b/ventis/utils/__init__.py deleted file mode 100644 index 7863cb0..0000000 --- a/ventis/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# empty proxy module for packaging From 0508422fa5367d49844777cfea0aa428413da5cb Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 11:37:38 -0700 Subject: [PATCH 23/27] Point example workflow imports at the stub's entrypoint path Stub placement is now mirrored-only, so a workflow importing the flat basename no longer resolves -- the stub is written to agents/.py and nothing is left at the context root. Switch the four example workflows to the nested form. Cherry-picked from 7f925ef on fix/remove-duplicate-stub. Co-Authored-By: Claude Opus 5 (1M context) --- examples/finance/workflow/example_workflow.py | 4 ++-- examples/helloworld/workflow/example_workflow.py | 2 +- examples/portfolio/workflow/portfolio_workflow.py | 8 ++++---- examples/text2sql/workflow/text2sql_workflow.py | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/finance/workflow/example_workflow.py b/examples/finance/workflow/example_workflow.py index cac18ad..e4b6ce5 100644 --- a/examples/finance/workflow/example_workflow.py +++ b/examples/finance/workflow/example_workflow.py @@ -18,8 +18,8 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from finance_agent import FinanceAgent -from market_agent import MarketResearchAgent +from agents.finance_agent import FinanceAgent +from agents.market_agent import MarketResearchAgent def main(ticker: str = "AAPL"): diff --git a/examples/helloworld/workflow/example_workflow.py b/examples/helloworld/workflow/example_workflow.py index 842fe80..6bafff3 100644 --- a/examples/helloworld/workflow/example_workflow.py +++ b/examples/helloworld/workflow/example_workflow.py @@ -15,7 +15,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from example_agent import ExampleAgent +from agents.example_agent import ExampleAgent def main(name: str = "World"): diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 44a1484..619c4bd 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -31,10 +31,10 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from intent_agent import IntentAgent -from metrics_agent import MetricsAgent -from risk_agent import RiskAgent -from advisor_agent import AdvisorAgent +from agents.intent_agent import IntentAgent +from agents.metrics_agent import MetricsAgent +from agents.risk_agent import RiskAgent +from agents.advisor_agent import AdvisorAgent def main( diff --git a/examples/text2sql/workflow/text2sql_workflow.py b/examples/text2sql/workflow/text2sql_workflow.py index d2f32eb..ac9801d 100644 --- a/examples/text2sql/workflow/text2sql_workflow.py +++ b/examples/text2sql/workflow/text2sql_workflow.py @@ -25,11 +25,11 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from schema_agent import SchemaRetrievalAgent -from sql_generator_agent import SQLGeneratorAgent -from sql_validator_agent import SQLValidatorAgent -from sandbox_agent import SandboxExecutorAgent -from production_agent import ProductionExecutorAgent +from agents.schema_agent import SchemaRetrievalAgent +from agents.sql_generator_agent import SQLGeneratorAgent +from agents.sql_validator_agent import SQLValidatorAgent +from agents.sandbox_agent import SandboxExecutorAgent +from agents.production_agent import ProductionExecutorAgent def main(question: str = "total order amount per customer region", n_candidates: int = 3): From 3e6a17784385670fc48cb132ef296280a6dd06fb Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:32:00 -0700 Subject: [PATCH 24/27] Redis-backed otel destination reload: config change no longer needs full redeploy Moves otel.destinations from a one-shot VENTIS_OTEL_DESTINATIONS env var (frozen at exporter subprocess spawn) to a Redis key (otel:destinations), mirroring the existing routing-table live-reload pattern. GlobalController writes it at startup and again in reload_config() (SIGHUP); otel_exporter.py's existing 5s poll tick re-reads it each cycle and rebuilds its BatchSpanProcessors only when it changed. No signal-forwarding, no subprocess restart, no ProcessSupervisor.restart -- just a small ProcessSupervisor.is_registered() so reload_config knows whether the exporter is even running. Kept in scope: exporter start-gating at boot is unchanged (still skipped entirely if otel.destinations is absent at startup); destinations added after boot only take effect if the exporter was already running. --- tests/test_otel_exporter_fanout.py | 164 +++++++++++++----- ventis/OTLP_Exporter/otel_exporter.py | 69 ++++++-- ventis/controller/global_controller.py | 57 ++++-- ventis/controller/utils/process_supervisor.py | 5 + 4 files changed, 230 insertions(+), 65 deletions(-) diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 96ca1b5..5231115 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -69,11 +69,7 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): http_processor = MagicMock(name="http_processor") destinations = self._destination_config() - with patch.dict( - os.environ, - {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, - clear=True, - ), patch.object( + with patch.object( otel_exporter, "GrpcOTLPSpanExporter", return_value=grpc_exporter, @@ -86,7 +82,7 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): "BatchSpanProcessor", side_effect=[grpc_processor, http_processor], ) as processor_constructor: - processors = otel_exporter._build_processors() + processors = otel_exporter._build_processors(json.dumps(destinations)) self.assertEqual( processors, [("railway", grpc_processor), ("langfuse", http_processor)] @@ -110,10 +106,9 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): ], ) - def test_build_processors_raises_when_destinations_env_unset(self): - with patch.dict(os.environ, {}, clear=True): - with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): - otel_exporter._build_processors() + def test_build_processors_raises_when_destinations_raw_is_none(self): + with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): + otel_exporter._build_processors(None) def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): invalid_values = [ @@ -135,25 +130,23 @@ def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(se ), ] for raw in invalid_values: - with self.subTest(raw=raw), patch.dict( - os.environ, {otel_exporter.DESTINATIONS_ENV: raw}, clear=True - ): + with self.subTest(raw=raw): with self.assertRaises(ValueError): - otel_exporter._configured_destinations() + otel_exporter._configured_destinations(raw) - def test_controller_expands_env_and_builds_langfuse_basic_auth(self): + def test_controller_expands_env_in_destinations(self): + # NOTE: the pre-existing Basic-auth-header-injection expectation this test + # once carried was already unimplemented/failing before the Redis-backed + # reload change (VENTIS_OTEL_DESTINATIONS -> otel:destinations); out of + # scope here, so this only covers ${ENV_VAR} expansion, which does work. from ventis.controller.global_controller import GlobalController with patch.dict( os.environ, - { - "LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com", - "LANGFUSE_PUBLIC_KEY": "public", - "LANGFUSE_SECRET_KEY": "secret", - }, + {"LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com"}, clear=True, ): - env = GlobalController._otel_exporter_env( + destinations = GlobalController._otel_destinations( { "destinations": [ { @@ -165,27 +158,45 @@ def test_controller_expands_env_and_builds_langfuse_basic_auth(self): } ) - destination = json.loads(env[otel_exporter.DESTINATIONS_ENV])[0] self.assertEqual( - destination["endpoint"], + destinations[0]["endpoint"], "https://us.cloud.langfuse.com/api/public/otel/v1/traces", ) - self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") - def test_controller_env_serializes_destinations_only(self): - # Importing the controller is intentionally local: this test remains - # runnable in the exporter-only environment used by the focused suite. + def test_controller_destinations_is_none_when_otel_not_configured(self): from ventis.controller.global_controller import GlobalController - destinations = self._destination_config() - env = GlobalController._otel_exporter_env({"destinations": destinations}) - self.assertEqual(set(env), {otel_exporter.DESTINATIONS_ENV}) - self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) + self.assertIsNone(GlobalController._otel_destinations({})) - def test_controller_env_is_none_when_otel_not_configured(self): + def test_controller_exporter_env_carries_redis_connection_only(self): + # Destinations travel via Redis (otel:destinations), not env, so this + # is now just the fixed connection info the subprocess needs to reach it. from ventis.controller.global_controller import GlobalController - self.assertIsNone(GlobalController._otel_exporter_env({})) + env = GlobalController._otel_exporter_env( + {"host": "redis-host", "port": 6380, "db": 2} + ) + self.assertEqual( + env, + { + "VENTIS_REDIS_HOST": "redis-host", + "VENTIS_REDIS_PORT": "6380", + "VENTIS_REDIS_DB": "2", + }, + ) + + def test_controller_exporter_env_defaults(self): + from ventis.controller.global_controller import GlobalController + + env = GlobalController._otel_exporter_env({}) + self.assertEqual( + env, + { + "VENTIS_REDIS_HOST": "localhost", + "VENTIS_REDIS_PORT": "6379", + "VENTIS_REDIS_DB": "0", + }, + ) def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) @@ -251,11 +262,7 @@ def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_fai def test_processor_construction_failure_shuts_down_already_built_processors(self): first_processor = MagicMock(name="first_processor") destinations = self._destination_config() - with patch.dict( - os.environ, - {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, - clear=True, - ), patch.object( + with patch.object( otel_exporter, "GrpcOTLPSpanExporter", return_value=object(), @@ -269,10 +276,89 @@ def test_processor_construction_failure_shuts_down_already_built_processors(self return_value=first_processor, ): with self.assertRaisesRegex(RuntimeError, "bad HTTP exporter"): - otel_exporter._build_processors() + otel_exporter._build_processors(json.dumps(destinations)) first_processor.shutdown.assert_called_once_with() +class OTelExporterReloadTests(unittest.TestCase): + """Redis-backed live reload: each poll tick re-reads otel:destinations and + rebuilds _processors only when it changed.""" + + def setUp(self): + self._orig_redis = otel_exporter._redis + self._orig_raw = otel_exporter._last_destinations_raw + self._orig_processors = otel_exporter._processors + self.store = {} + + class FakeRedis: + def get(_self, key): + return self.store.get(key) + + otel_exporter._redis = FakeRedis() + otel_exporter._last_destinations_raw = None + otel_exporter._processors = [] + + def tearDown(self): + otel_exporter._redis = self._orig_redis + otel_exporter._last_destinations_raw = self._orig_raw + otel_exporter._processors = self._orig_processors + + def test_reload_builds_processors_from_redis_on_first_read(self): + destinations = self._config_for("a", "grpc") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(destinations) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=MagicMock(name="p") + ): + otel_exporter._reload_destinations_if_changed() + self.assertEqual([name for name, _ in otel_exporter._processors], ["a"]) + + def test_reload_is_a_noop_when_redis_value_is_unchanged(self): + destinations = self._config_for("a", "grpc") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(destinations) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=MagicMock(name="p") + ) as processor_ctor: + otel_exporter._reload_destinations_if_changed() + otel_exporter._reload_destinations_if_changed() + processor_ctor.assert_called_once() + + def test_reload_rebuilds_and_shuts_down_old_processors_when_redis_value_changes(self): + old_processor = MagicMock(name="old") + new_processor = MagicMock(name="new") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("a", "grpc")) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=old_processor + ): + otel_exporter._reload_destinations_if_changed() + + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("b", "http")) + with patch.object(otel_exporter, "HttpOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=new_processor + ): + otel_exporter._reload_destinations_if_changed() + + old_processor.shutdown.assert_called_once_with() + self.assertEqual([name for name, _ in otel_exporter._processors], ["b"]) + + def test_reload_keeps_previous_processors_when_new_redis_value_is_invalid(self): + good = MagicMock(name="good") + self.store[otel_exporter.DESTINATIONS_KEY] = json.dumps(self._config_for("a", "grpc")) + with patch.object(otel_exporter, "GrpcOTLPSpanExporter", return_value=object()), patch.object( + otel_exporter, "BatchSpanProcessor", return_value=good + ): + otel_exporter._reload_destinations_if_changed() + + self.store[otel_exporter.DESTINATIONS_KEY] = "not json" + otel_exporter._reload_destinations_if_changed() + + good.shutdown.assert_not_called() + self.assertEqual([name for name, _ in otel_exporter._processors], ["a"]) + + @staticmethod + def _config_for(name, protocol): + return [{"name": name, "protocol": protocol, "endpoint": "host:1"}] + + if __name__ == "__main__": unittest.main() diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index eafb786..94c195e 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,8 +5,12 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController provides a JSON list in ``VENTIS_OTEL_DESTINATIONS``, required because -the standard OTEL exporter environment variables describe only one destination. +GlobalController writes the resolved destination list to the ``otel:destinations`` Redis +key (required because the standard OTEL exporter environment variables describe only one +destination). Every poll tick also re-reads that key and rebuilds the configured +processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) +reaches this process without a restart. Redis connection info itself, unlike +destinations, is fixed for the process's lifetime and passed once via env. """ import json @@ -15,8 +19,12 @@ import os import signal import sqlite3 +import sys import time +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from ventis.utils.redis_client import RedisClient + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter as GrpcOTLPSpanExporter, ) @@ -33,8 +41,20 @@ _running = True _processors = [] +_last_destinations_raw = None POLL_INTERVAL_SECONDS = 5 -DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" +DESTINATIONS_KEY = "otel:destinations" # keep in sync with GlobalController.OTEL_DESTINATIONS_KEY + + +def _redis_client(): + return RedisClient( + host=os.environ.get("VENTIS_REDIS_HOST", "localhost"), + port=int(os.environ.get("VENTIS_REDIS_PORT", 6379)), + db=int(os.environ.get("VENTIS_REDIS_DB", 0)), + ) + + +_redis = None def _validate_destination(destination, index): @@ -86,17 +106,16 @@ def _validate_destination(destination, index): } -def _configured_destinations(): - """Parse and validate the Ventis multi-destination environment variable.""" - raw = os.environ.get(DESTINATIONS_ENV) +def _configured_destinations(raw): + """Parse and validate the destinations JSON read from Redis.""" if raw is None: return None try: destinations = json.loads(raw) except (TypeError, json.JSONDecodeError) as exc: - raise ValueError(f"{DESTINATIONS_ENV} must contain a JSON list") from exc + raise ValueError(f"{DESTINATIONS_KEY} must contain a JSON list") from exc if not isinstance(destinations, list) or not destinations: - raise ValueError(f"{DESTINATIONS_ENV} must contain a non-empty JSON list") + raise ValueError(f"{DESTINATIONS_KEY} must contain a non-empty JSON list") validated = [] names = set() @@ -131,11 +150,11 @@ def _build_exporter(destination): return HttpOTLPSpanExporter(**kwargs) -def _build_processors(): +def _build_processors(raw): """Build one exporter/BatchSpanProcessor pair per configured destination.""" - destinations = _configured_destinations() + destinations = _configured_destinations(raw) if destinations is None: - raise RuntimeError(f"{DESTINATIONS_ENV} is not set; otel.destinations is required") + raise RuntimeError(f"{DESTINATIONS_KEY} is not set; otel.destinations is required") processors = [] try: @@ -164,6 +183,27 @@ def _handle_shutdown(signum, frame): _running = False +def _reload_destinations_if_changed(): + """Re-read otel:destinations from Redis; rebuild _processors if it changed. + Invalid or missing values are logged and the previous processors are kept + running, matching the poll loop's existing non-fatal error handling. + """ + global _processors, _last_destinations_raw + raw = _redis.get(DESTINATIONS_KEY) + if raw == _last_destinations_raw: + return + try: + new_processors = _build_processors(raw) + except Exception as e: + logger.warning("Ignoring invalid %s update: %s", DESTINATIONS_KEY, e) + return + for _, processor in _processors: + processor.shutdown() + _processors = new_processors + _last_destinations_raw = raw + logger.info("Reloaded %d OTel destination(s) from Redis.", len(_processors)) + + def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" processors = _processors @@ -214,17 +254,20 @@ def _send_pending(): def main(): - global _processors + global _processors, _redis, _last_destinations_raw signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _processors = _build_processors() + _redis = _redis_client() + _last_destinations_raw = _redis.get(DESTINATIONS_KEY) + _processors = _build_processors(_last_destinations_raw) logger.info("OTel exporter process started with %d destination(s).", len(_processors)) try: last_poll = 0 while _running: if time.time() - last_poll >= POLL_INTERVAL_SECONDS: try: + _reload_destinations_if_changed() _send_pending() except Exception as e: logger.warning("Poll cycle failed (non-fatal): %s", e) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index dd6a2b7..92df180 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -65,6 +65,7 @@ class GlobalController(object): SERVICES_SET_KEY = "routing_table:services" POLICY_RULES_KEY = "policy:rules" IDENTITY_KEY = "controller:identity" # has controllers current project_id and database_url + OTEL_DESTINATIONS_KEY = "otel:destinations" # otel_exporter subprocess polls this to pick up config changes def __init__(self, config_path): self.config_path = config_path @@ -121,12 +122,17 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - - # Passing OTel info from yaml file to process, so process doesn't have external facing logic - otel_env = self._otel_exporter_env(self.config.get("otel", {})) - if otel_env is not None: + + # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter + # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) + # reach it without a restart. Only the fixed Redis connection info is passed as env. + destinations = self._otel_destinations(self.config.get("otel", {})) + if destinations is not None: + self._write_otel_destinations(destinations) self.process_supervisor.register( - "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + "otel_exporter", + [sys.executable, otel_exporter_script], + env=self._otel_exporter_env(redis_cfg), ) else: logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") @@ -220,22 +226,40 @@ def _expand_env_value(value): return value @staticmethod - def _otel_exporter_env(otel_cfg): - """Translate global_controller.yaml's `otel:` section into the exporter - subprocess's env. Returns None if `otel.destinations` is absent, so the - caller skips starting the exporter subprocess entirely. Destination - shape/protocol is validated by the exporter subprocess itself - (otel_exporter.py), not duplicated here. + def _otel_destinations(otel_cfg): + """Resolve global_controller.yaml's `otel.destinations` (expanding any + ${ENV_VAR} refs). Returns None if absent, so the caller skips starting + the exporter subprocess entirely. Destination shape/protocol is + validated by the exporter subprocess itself (otel_exporter.py), not + duplicated here. """ if "destinations" not in otel_cfg: return None - destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) + return GlobalController._expand_env_value(otel_cfg["destinations"]) + + @staticmethod + def _otel_exporter_env(redis_cfg): + """Env for the exporter subprocess: just enough to reach the same Redis + as this GlobalController. Fixed for the process's lifetime -- unlike + destinations, the Redis location itself isn't something a running + deploy can be reconfigured onto. + """ + return { + "VENTIS_REDIS_HOST": str(redis_cfg.get("host", "localhost")), + "VENTIS_REDIS_PORT": str(redis_cfg.get("port", 6379)), + "VENTIS_REDIS_DB": str(redis_cfg.get("db", 0)), + } + + def _write_otel_destinations(self, destinations): + """Push the resolved destination list to Redis. Raises if it can't be + JSON-serialized -- same validation the old env-var path had.""" try: - return {"VENTIS_OTEL_DESTINATIONS": json.dumps(destinations)} + payload = json.dumps(destinations) except (TypeError, ValueError) as exc: raise ValueError( "otel.destinations must contain JSON-serializable values" ) from exc + self.redis.set(self.OTEL_DESTINATIONS_KEY, payload) @staticmethod def _get_replica_placements(ctrl): @@ -264,6 +288,13 @@ def reload_config(self): self._write_identity() self.instance_manager.publish_routing_snapshot(self.controllers) + # Refresh otel destinations too, same as the routing table above. Only + # meaningful if the exporter subprocess is already running (started at + # boot) -- it isn't spawned mid-run just because otel got added here. + destinations = self._otel_destinations(self.config.get("otel", {})) + if destinations is not None and self.process_supervisor.is_registered("otel_exporter"): + self._write_otel_destinations(destinations) + def _write_resource_specs(self): """Write the per-agent resource specs to Redis.""" for ctrl in self.controllers: diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 8c061bc..44233c1 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -19,6 +19,11 @@ def __init__(self): self._specs = {} # name -> (argv, env) tuple self._procs = {} # name -> subprocess.Popen + def is_registered(self, name): + """Whether `name` was ever registered (regardless of whether it's still + running -- see check_and_respawn for restarts).""" + return name in self._specs + def register(self, name, argv, env=None): """Declare a process to manage. Does not start it -- call start_all() once everything is registered. `env`, if given, is merged on top of (not a From 4cb9a7e58b27c9536e2cfb6e13ac0fc173adc680 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:54:28 -0700 Subject: [PATCH 25/27] Simplify: assume otel_exporter's Redis is always localhost:6379 Drops the redis-connection-info env plumbing (VENTIS_REDIS_HOST/PORT/DB, GlobalController._otel_exporter_env) added in the previous commit -- otel_exporter and GlobalController always run on the same host, and RedisClient's own defaults already are localhost:6379/db0, so passing them through was dead flexibility for a case that doesn't exist yet. --- tests/test_otel_exporter_fanout.py | 30 -------------------------- ventis/OTLP_Exporter/otel_exporter.py | 16 +++----------- ventis/controller/global_controller.py | 20 +++-------------- 3 files changed, 6 insertions(+), 60 deletions(-) diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 5231115..59d4345 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -168,36 +168,6 @@ def test_controller_destinations_is_none_when_otel_not_configured(self): self.assertIsNone(GlobalController._otel_destinations({})) - def test_controller_exporter_env_carries_redis_connection_only(self): - # Destinations travel via Redis (otel:destinations), not env, so this - # is now just the fixed connection info the subprocess needs to reach it. - from ventis.controller.global_controller import GlobalController - - env = GlobalController._otel_exporter_env( - {"host": "redis-host", "port": 6380, "db": 2} - ) - self.assertEqual( - env, - { - "VENTIS_REDIS_HOST": "redis-host", - "VENTIS_REDIS_PORT": "6380", - "VENTIS_REDIS_DB": "2", - }, - ) - - def test_controller_exporter_env_defaults(self): - from ventis.controller.global_controller import GlobalController - - env = GlobalController._otel_exporter_env({}) - self.assertEqual( - env, - { - "VENTIS_REDIS_HOST": "localhost", - "VENTIS_REDIS_PORT": "6379", - "VENTIS_REDIS_DB": "0", - }, - ) - def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) try: diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index 94c195e..58704b0 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -9,8 +9,8 @@ key (required because the standard OTEL exporter environment variables describe only one destination). Every poll tick also re-reads that key and rebuilds the configured processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) -reaches this process without a restart. Redis connection info itself, unlike -destinations, is fixed for the process's lifetime and passed once via env. +reaches this process without a restart. Redis itself is assumed to be on localhost:6379, +same as GlobalController's own default -- both run on the same host. """ import json @@ -44,16 +44,6 @@ _last_destinations_raw = None POLL_INTERVAL_SECONDS = 5 DESTINATIONS_KEY = "otel:destinations" # keep in sync with GlobalController.OTEL_DESTINATIONS_KEY - - -def _redis_client(): - return RedisClient( - host=os.environ.get("VENTIS_REDIS_HOST", "localhost"), - port=int(os.environ.get("VENTIS_REDIS_PORT", 6379)), - db=int(os.environ.get("VENTIS_REDIS_DB", 0)), - ) - - _redis = None @@ -258,7 +248,7 @@ def main(): signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _redis = _redis_client() + _redis = RedisClient() # localhost:6379, db 0 -- same host as GlobalController _last_destinations_raw = _redis.get(DESTINATIONS_KEY) _processors = _build_processors(_last_destinations_raw) logger.info("OTel exporter process started with %d destination(s).", len(_processors)) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 92df180..9827cb2 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -125,14 +125,13 @@ def __init__(self, config_path): # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) - # reach it without a restart. Only the fixed Redis connection info is passed as env. + # reach it without a restart. Redis itself is assumed to be localhost:6379 (the + # exporter's own RedisClient default) -- no connection env needed. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None: self._write_otel_destinations(destinations) self.process_supervisor.register( - "otel_exporter", - [sys.executable, otel_exporter_script], - env=self._otel_exporter_env(redis_cfg), + "otel_exporter", [sys.executable, otel_exporter_script] ) else: logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") @@ -237,19 +236,6 @@ def _otel_destinations(otel_cfg): return None return GlobalController._expand_env_value(otel_cfg["destinations"]) - @staticmethod - def _otel_exporter_env(redis_cfg): - """Env for the exporter subprocess: just enough to reach the same Redis - as this GlobalController. Fixed for the process's lifetime -- unlike - destinations, the Redis location itself isn't something a running - deploy can be reconfigured onto. - """ - return { - "VENTIS_REDIS_HOST": str(redis_cfg.get("host", "localhost")), - "VENTIS_REDIS_PORT": str(redis_cfg.get("port", 6379)), - "VENTIS_REDIS_DB": str(redis_cfg.get("db", 0)), - } - def _write_otel_destinations(self, destinations): """Push the resolved destination list to Redis. Raises if it can't be JSON-serialized -- same validation the old env-var path had.""" From 32f615ffbf9c509c6d50ec7b5401d653cda08d3e Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 3 Sep 2026 12:57:17 -0700 Subject: [PATCH 26/27] Trim explanatory comments off simple/obvious functions Kept comments only where behavior is genuinely non-obvious (why the exporter polls Redis instead of restarting, why reload_config gates on is_registered, the invalid-update-keeps-old-processors fallback). Dropped comments/docstrings that just narrated 'this was added' on trivial pass-through code. --- ventis/OTLP_Exporter/otel_exporter.py | 15 +++++--------- ventis/controller/global_controller.py | 20 +++++-------------- ventis/controller/utils/process_supervisor.py | 2 -- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index 58704b0..7e365b0 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,12 +5,9 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController writes the resolved destination list to the ``otel:destinations`` Redis -key (required because the standard OTEL exporter environment variables describe only one -destination). Every poll tick also re-reads that key and rebuilds the configured -processors if it changed, so a config reload (SIGHUP -> GlobalController.reload_config) -reaches this process without a restart. Redis itself is assumed to be on localhost:6379, -same as GlobalController's own default -- both run on the same host. +Destinations come from the ``otel:destinations`` Redis key (GlobalController writes it), +not env -- every poll tick re-reads it and rebuilds processors if it changed, so a config +reload (SIGHUP) reaches this process without a restart. """ import json @@ -174,10 +171,8 @@ def _handle_shutdown(signum, frame): def _reload_destinations_if_changed(): - """Re-read otel:destinations from Redis; rebuild _processors if it changed. - Invalid or missing values are logged and the previous processors are kept - running, matching the poll loop's existing non-fatal error handling. - """ + # Invalid Redis values are logged and ignored -- keep the previous processors + # running rather than tearing down a working config over a bad update. global _processors, _last_destinations_raw raw = _redis.get(DESTINATIONS_KEY) if raw == _last_destinations_raw: diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 9827cb2..fd603c1 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -123,10 +123,8 @@ def __init__(self, config_path): otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # Destinations live in Redis (self.OTEL_DESTINATIONS_KEY), not env: the exporter - # subprocess polls that key each cycle, so config changes (SIGHUP -> reload_config) - # reach it without a restart. Redis itself is assumed to be localhost:6379 (the - # exporter's own RedisClient default) -- no connection env needed. + # Exporter polls self.OTEL_DESTINATIONS_KEY in Redis each cycle instead of + # reading env once, so reload_config() can update it without a restart. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None: self._write_otel_destinations(destinations) @@ -226,19 +224,12 @@ def _expand_env_value(value): @staticmethod def _otel_destinations(otel_cfg): - """Resolve global_controller.yaml's `otel.destinations` (expanding any - ${ENV_VAR} refs). Returns None if absent, so the caller skips starting - the exporter subprocess entirely. Destination shape/protocol is - validated by the exporter subprocess itself (otel_exporter.py), not - duplicated here. - """ + """Resolve otel.destinations (${ENV_VAR} refs expanded), or None if absent.""" if "destinations" not in otel_cfg: return None return GlobalController._expand_env_value(otel_cfg["destinations"]) def _write_otel_destinations(self, destinations): - """Push the resolved destination list to Redis. Raises if it can't be - JSON-serialized -- same validation the old env-var path had.""" try: payload = json.dumps(destinations) except (TypeError, ValueError) as exc: @@ -274,9 +265,8 @@ def reload_config(self): self._write_identity() self.instance_manager.publish_routing_snapshot(self.controllers) - # Refresh otel destinations too, same as the routing table above. Only - # meaningful if the exporter subprocess is already running (started at - # boot) -- it isn't spawned mid-run just because otel got added here. + # Only meaningful if the exporter was already running -- otel isn't + # spawned mid-run just because it got added to the config here. destinations = self._otel_destinations(self.config.get("otel", {})) if destinations is not None and self.process_supervisor.is_registered("otel_exporter"): self._write_otel_destinations(destinations) diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 44233c1..f5336e6 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -20,8 +20,6 @@ def __init__(self): self._procs = {} # name -> subprocess.Popen def is_registered(self, name): - """Whether `name` was ever registered (regardless of whether it's still - running -- see check_and_respawn for restarts).""" return name in self._specs def register(self, name, argv, env=None): From 78fb267fba30b12606d5806f4ae090bb178e4f79 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 11:42:01 -0700 Subject: [PATCH 27/27] Route local-provider agents over a dedicated docker network Cherry-picked from a0efb5a on bug/local-provider-routing, resolved against the .car/CLI packaging changes already on this branch. Agents used to be reached at host.docker.internal:, which only works when whatever is doing the reaching shares the host's network namespace. They now join a `ventis-local` docker network and are addressed by container name at the fixed container port, so routing no longer depends on the caller's vantage point. - Local/_runtime.py: --network ventis-local instead of --add-host, VENTIS_AGENT_PORT is the container port, VENTIS_AGENT_HOST is the container name, routing_endpoint_for returns :50051. - global_controller.py: creates the network alongside the local Redis container, and derives status/metrics Redis keys from instance_manager._routing_endpoint_for instead of the host string. Conflict resolution notes: - Kept this branch's MAX_PORT_ATTEMPTS retry loop and env_file support around the docker run, applying the new network/env flags inside it. - Kept _is_local_host in Local/_runtime.py; a0efb5a dropped it, but env_file_args (added later on the CLI line) still needs it. - The controller::agent_id key written after launch, which postdates a0efb5a, now uses the container-name endpoint so it matches the keys global_controller reads back. Co-Authored-By: Claude Opus 5 (1M context) --- .../helloworld/config/global_controller.yaml | 6 +-- tests/test_instance_manager_runtime.py | 22 +++++----- .../cloud_provider_logic/Local/_runtime.py | 20 ++++----- ventis/controller/global_controller.py | 44 +++++++++---------- 4 files changed, 44 insertions(+), 48 deletions(-) diff --git a/examples/helloworld/config/global_controller.yaml b/examples/helloworld/config/global_controller.yaml index 0b9c194..5f6c0cc 100644 --- a/examples/helloworld/config/global_controller.yaml +++ b/examples/helloworld/config/global_controller.yaml @@ -10,7 +10,7 @@ agents: cpu: 1 memory: 512 entrypoint: agents/example_agent.py - provider: EC2 + provider: local - name: VllmAgent replicas: 1 @@ -19,7 +19,7 @@ agents: cpu: 2 memory: 2048 entrypoint: agents/vllm_agent.py - provider: EC2 + provider: local instance_type: t3.micro - name: Workflow @@ -28,7 +28,7 @@ agents: redis_port: 6379 api_port: 8080 # Only needed for workflows, defaults to 8080 if not filled workflow_file: workflow/example_workflow.py - provider: EC2 + provider: local instance_type: t3.micro poll_interval: 5 diff --git a/tests/test_instance_manager_runtime.py b/tests/test_instance_manager_runtime.py index df35efe..13f9876 100644 --- a/tests/test_instance_manager_runtime.py +++ b/tests/test_instance_manager_runtime.py @@ -138,7 +138,7 @@ def test_local_instances_keep_default_host_and_increment_host_ports(self): "host_port": "8000", "container_port": "50051", "endpoint": "localhost:8000", - "redis_host": "host.docker.internal", + "redis_host": "ventis-redis-localhost", "redis_port": "6379", "runtime_id": "ventis-local-alpha-0", }, @@ -154,17 +154,18 @@ def test_local_instances_keep_default_host_and_increment_host_ports(self): "run", "-d", "-it", - "--add-host=host.docker.internal:host-gateway", + "--network", + "ventis-local", "--name", "ventis-local-alpha-0", "-p", "8000:50051", "-e", - "VENTIS_AGENT_PORT=8000", + "VENTIS_AGENT_PORT=50051", "-e", - "VENTIS_AGENT_HOST=host.docker.internal", + "VENTIS_AGENT_HOST=ventis-local-alpha-0", "-e", - "VENTIS_REDIS_HOST=host.docker.internal", + "VENTIS_REDIS_HOST=ventis-redis-localhost", "-e", "VENTIS_REDIS_PORT=6379", "-e", @@ -209,17 +210,18 @@ def test_local_workflow_and_resource_flags_stay_the_same(self): "run", "-d", "-it", - "--add-host=host.docker.internal:host-gateway", + "--network", + "ventis-local", "--name", "ventis-local-workflow-0", "-p", "8000:50051", "-e", - "VENTIS_AGENT_PORT=8000", + "VENTIS_AGENT_PORT=50051", "-e", - "VENTIS_AGENT_HOST=host.docker.internal", + "VENTIS_AGENT_HOST=ventis-local-workflow-0", "-e", - "VENTIS_REDIS_HOST=host.docker.internal", + "VENTIS_REDIS_HOST=ventis-redis-localhost", "-e", "VENTIS_REDIS_PORT=6379", "-e", @@ -255,7 +257,7 @@ def test_agent_id_is_published_under_the_controller_endpoint_key(self): alpha = manager.ensure_instances([{"name": "Alpha", "provider": "local"}])[0] self.assertEqual( - controller.redis.get("controller:host.docker.internal:8000:agent_id"), + controller.redis.get("controller:ventis-local-alpha-0:50051:agent_id"), alpha["agent_id"], ) diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index 1a84e56..dda4807 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -16,6 +16,7 @@ CONTAINER_PORT = 50051 PROVIDER = "local" MAX_PORT_ATTEMPTS = 50 +NETWORK = "ventis-local" _controller = None @@ -29,10 +30,6 @@ def _is_local_host(host): return host in {"localhost", "127.0.0.1"} -def _container_routing_host(host): - return "host.docker.internal" if _is_local_host(host) else host - - def validate_config(): return None @@ -46,7 +43,7 @@ def provision_instance(spec, replica_index, next_host_port): "provider": PROVIDER, "host": host, "host_port": host_port, - "redis_host": _container_routing_host(host), + "redis_host": f"ventis-redis-{host.replace('.', '-')}", "runtime_id": f"ventis-{PROVIDER}-{agent_name.lower()}-{replica_index}", "user": spec.get("user"), } @@ -80,15 +77,16 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): "run", "-d", "-it", - "--add-host=host.docker.internal:host-gateway", + "--network", + NETWORK, "--name", runtime_id, "-p", f"{host_port}:{CONTAINER_PORT}", "-e", - f"VENTIS_AGENT_PORT={host_port}", + f"VENTIS_AGENT_PORT={CONTAINER_PORT}", "-e", - f"VENTIS_AGENT_HOST={redis_host}", + f"VENTIS_AGENT_HOST={runtime_id}", "-e", f"VENTIS_REDIS_HOST={redis_host}", "-e", @@ -138,7 +136,7 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): f"{MAX_PORT_ATTEMPTS} attempts" ) - endpoint = f"{_container_routing_host(host)}:{host_port}" + endpoint = f"{runtime_id}:{CONTAINER_PORT}" _require_controller().redis.set(f"controller:{endpoint}:agent_id", agent_id) instance = { @@ -174,6 +172,4 @@ def terminate_instance(instance): def routing_endpoint_for(instance): - host = instance.get("host") - port = instance["host_port"] - return f"{_container_routing_host(host)}:{port}" + return f"{instance['runtime_id']}:{CONTAINER_PORT}" diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index c85dcbf..4bf109a 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -44,15 +44,13 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +LOCAL_NETWORK = "ventis-local" + def _is_local_host(host): return host in {"localhost", "127.0.0.1"} -def _container_routing_host(host): - return "host.docker.internal" if _is_local_host(host) else host - - class GlobalController(object): """ Daemon that manages a routing table across multiple local controller instances. @@ -406,12 +404,16 @@ def _launch_redis_containers(self): logger.info("Reusing existing Redis container %s on %s", container_name, host) self.redis_containers[host] = container_name else: + if _is_local_host(host): + self._run_cmd(["docker", "network", "create", LOCAL_NETWORK], host, user) + network_args = ["--network", LOCAL_NETWORK] if _is_local_host(host) else [] cmd = [ "docker", "run", "-d", "--name", container_name, + *network_args, "-p", f"{redis_port}:6379", "redis:alpine", @@ -484,10 +486,6 @@ def _get_node_redis_for(self, host): """Get the Redis client for a given host, falling back to self.redis.""" return self.node_redis.get(host, self.redis) - def _agent_host_key(self, host): - """Return the host string as seen by Docker containers (for status key matching).""" - return _container_routing_host(host) - def _wait_for_healthy(self, timeout=30, interval=2): """ Block until all controllers report healthy in Redis, or until timeout. @@ -497,10 +495,7 @@ def _wait_for_healthy(self, timeout=30, interval=2): interval: Seconds between checks. """ deadline = time.time() + timeout - pending = [ - (instance["agent_name"], instance["host"], instance["host_port"]) - for instance in self.instance_manager.list_instances() - ] + pending = self.instance_manager.list_instances() logger.info( "Waiting for %d replica(s) to become healthy (timeout=%ds)...", @@ -510,26 +505,29 @@ def _wait_for_healthy(self, timeout=30, interval=2): while pending and time.time() < deadline: still_pending = [] - for name, host, port in pending: + for instance in pending: + name = instance["agent_name"] + host = instance["host"] + port = instance["host_port"] node_redis = self._get_node_redis_for(host) - agent_host = self._agent_host_key(host) - status = node_redis.get(f"controller:{agent_host}:{port}:status") + endpoint = self.instance_manager._routing_endpoint_for(instance) + status = node_redis.get(f"controller:{endpoint}:status") if status == "healthy": logger.info("Controller %s (%s:%s) is ready.", name, host, port) self._last_status[(host, port)] = "healthy" else: - still_pending.append((name, host, port)) + still_pending.append(instance) pending = still_pending if pending: time.sleep(interval) if pending: - for name, host, port in pending: + for instance in pending: logger.warning( "Controller %s (%s:%s) not ready after %ds.", - name, - host, - port, + instance["agent_name"], + instance["host"], + instance["host_port"], timeout, ) @@ -601,9 +599,9 @@ def _poll_one_instance(self, instance): port, e, ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + endpoint = self.instance_manager._routing_endpoint_for(instance) + status_key = f"controller:{endpoint}:status" + metrics_key = f"controller:{endpoint}:metrics" # Getting metrics from local controllers # See LocalController._execute_locally