diff --git a/ventis/llm_proxy/README.md b/ventis/llm_proxy/README.md index 873144d..873510b 100644 --- a/ventis/llm_proxy/README.md +++ b/ventis/llm_proxy/README.md @@ -65,6 +65,39 @@ boto3.client("bedrock-runtime").invoke_model( "messages": [{"role": "user", "content": "hi"}]})) ``` +## Health checks + +`GET /healthz` with no params just reports which provider adapters are +compiled in — it doesn't call any upstream: + +```json +{"status": "ok", "providers": ["anthropic", "bedrock", "openai"]} +``` + +To actually verify a credential can reach a specific model, pass +`_model=`: + +```bash +curl 'localhost:8080/healthz?openai_model=gpt-4o-mini&anthropic_model=claude-3-5-sonnet-20241022' +``` + +```json +{ + "status": "degraded", + "providers": ["anthropic", "bedrock", "openai"], + "models": { + "openai": {"model": "gpt-4o-mini", "ok": true}, + "anthropic": {"model": "claude-3-5-sonnet-20241022", "ok": false, "error": "upstream returned 404"} + } +} +``` + +OpenAI/Anthropic checks are a free `GET /v1/models/{id}` — confirms both the +key and the model in one request. Bedrock's check (`bedrock_model=`) only +confirms the model exists in the configured region's catalog; Bedrock has no +free way to confirm your account actually has invoke access granted for that +model — that requires a real (billed) `invoke_model` call. + ## Configuration (env vars) | Var | Default | Purpose | diff --git a/ventis/llm_proxy/app.py b/ventis/llm_proxy/app.py index b2e3b09..655f9cd 100644 --- a/ventis/llm_proxy/app.py +++ b/ventis/llm_proxy/app.py @@ -27,7 +27,20 @@ def create_app(cfg: Config = None) -> Flask: @app.route("/healthz", methods=["GET"]) def healthz(): - return jsonify(status="ok", providers=sorted(registry.keys())) + # Deep model checks are opt-in via `_model=` query params + # so a plain GET /healthz stays a cheap, zero-upstream-call probe. + checks = {} + for name, prov in registry.items(): + model_id = request.args.get(f"{name}_model") + if model_id: + checks[name] = prov.check_model(model_id) + + result = {"status": "ok", "providers": sorted(registry.keys())} + if checks: + result["models"] = checks + if not all(c["ok"] for c in checks.values()): + result["status"] = "degraded" + return jsonify(**result) @app.route("//", methods=ALL_METHODS) def dispatch(provider, subpath): diff --git a/ventis/llm_proxy/providers/anthropic.py b/ventis/llm_proxy/providers/anthropic.py index 33e14aa..b3d4a59 100644 --- a/ventis/llm_proxy/providers/anthropic.py +++ b/ventis/llm_proxy/providers/anthropic.py @@ -17,3 +17,14 @@ def target(self, req, subpath, body): headers=headers, params=req.args.to_dict(flat=True), ) + + def _model_url(self, model_id): + return f"{self.cfg.anthropic.upstream_base}/v1/models/{model_id}" + + def _model_check_headers(self): + if not self.cfg.anthropic.api_key: + return None + return { + "x-api-key": self.cfg.anthropic.api_key, + "anthropic-version": "2023-06-01", + } diff --git a/ventis/llm_proxy/providers/base.py b/ventis/llm_proxy/providers/base.py index ef5db41..24b39ec 100644 --- a/ventis/llm_proxy/providers/base.py +++ b/ventis/llm_proxy/providers/base.py @@ -10,7 +10,7 @@ import json from dataclasses import dataclass, field -from typing import Dict, Iterable, List, Tuple +from typing import Dict, Iterable, List, Optional, Tuple import requests @@ -72,6 +72,14 @@ def __init__(self, cfg): def forward(self, req, subpath: str, body: bytes) -> ProxyResponse: raise NotImplementedError + def check_model(self, model_id: str) -> dict: + """Verify a specific model is reachable with this provider's credentials. + + Used by ``/healthz?_model=``. Returns a JSON-serializable + dict with at least ``model`` and ``ok``. + """ + raise NotImplementedError + class HttpProvider(Provider): """Providers that are a straight HTTP reverse-proxy (OpenAI, Anthropic).""" @@ -94,3 +102,32 @@ def forward(self, req, subpath, body): headers=filter_response_headers(resp.headers), content=resp.content, ) + + # -- model existence/access check, shared by OpenAI + Anthropic ------- + # + # Both providers expose a free `GET /v1/models/{id}` that succeeds only if + # the credential is valid *and* the model exists/is accessible to the + # account, so a real request is cheap enough to make on every check. + + def _model_url(self, model_id: str) -> str: + raise NotImplementedError + + def _model_check_headers(self) -> Optional[Dict[str, str]]: + """Auth headers for the model check, or None if no key is configured.""" + raise NotImplementedError + + def check_model(self, model_id: str) -> dict: + headers = self._model_check_headers() + if headers is None: + return {"model": model_id, "ok": False, "error": "no API key configured"} + try: + resp = requests.get( + self._model_url(model_id), + headers=headers, + timeout=(self.cfg.connect_timeout, self.cfg.read_timeout), + ) + except requests.RequestException as exc: + return {"model": model_id, "ok": False, "error": str(exc)} + if resp.status_code == 200: + return {"model": model_id, "ok": True} + return {"model": model_id, "ok": False, "error": f"upstream returned {resp.status_code}"} diff --git a/ventis/llm_proxy/providers/bedrock.py b/ventis/llm_proxy/providers/bedrock.py index f0efddc..8a4f613 100644 --- a/ventis/llm_proxy/providers/bedrock.py +++ b/ventis/llm_proxy/providers/bedrock.py @@ -33,6 +33,10 @@ def __init__(self, cfg): region_name=cfg.bedrock_region, endpoint_url=f"https://{cfg.bedrock_upstream_host}" ) + # Control-plane client (not bedrock-runtime) — only this one exposes + # model catalog metadata, and it isn't affected by + # AWS_ENDPOINT_URL_BEDROCK_RUNTIME so needs no endpoint override. + self._control_client = boto3.client("bedrock", region_name=cfg.bedrock_region) def forward(self, req, subpath, body): model_id, op = self._parse(subpath) @@ -90,6 +94,27 @@ def forward(self, req, subpath, body): + def check_model(self, model_id: str) -> dict: + """Confirm ``model_id`` exists in this region's model catalog. + + This only proves the model ID is valid for the configured region — it + does NOT confirm this account/role has been granted invoke access, + since that's not queryable without an actual (billed) invoke call. + """ + try: + self._control_client.get_foundation_model(modelIdentifier=model_id) + except ClientError as exc: + return { + "model": model_id, + "ok": False, + "error": exc.response.get("Error", {}).get("Message", str(exc)), + } + return { + "model": model_id, + "ok": True, + "note": "confirms the model exists in this region; does not confirm invoke access is granted", + } + @staticmethod def _parse(subpath): # subpath looks like "model//"; the modelId may itself diff --git a/ventis/llm_proxy/providers/openai.py b/ventis/llm_proxy/providers/openai.py index 67457eb..a250ab3 100644 --- a/ventis/llm_proxy/providers/openai.py +++ b/ventis/llm_proxy/providers/openai.py @@ -16,3 +16,11 @@ def target(self, req, subpath, body): headers=headers, params=req.args.to_dict(flat=True), ) + + def _model_url(self, model_id): + return f"{self.cfg.openai.upstream_base}/v1/models/{model_id}" + + def _model_check_headers(self): + if not self.cfg.openai.api_key: + return None + return {"Authorization": f"Bearer {self.cfg.openai.api_key}"} diff --git a/ventis/llm_proxy/tests/__init__.py b/ventis/llm_proxy/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ventis/llm_proxy/tests/test_healthz.py b/ventis/llm_proxy/tests/test_healthz.py new file mode 100644 index 0000000..e43f5b2 --- /dev/null +++ b/ventis/llm_proxy/tests/test_healthz.py @@ -0,0 +1,118 @@ +"""Tests for /healthz, including the opt-in per-model deep check (CAN-287).""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from botocore.exceptions import ClientError + +from ventis.llm_proxy.app import create_app +from ventis.llm_proxy.config import Config, ProviderConfig + + +def make_cfg(openai_key="sk-openai", anthropic_key="sk-ant"): + return Config( + host="127.0.0.1", + port=8080, + connect_timeout=1.0, + read_timeout=1.0, + openai=ProviderConfig(upstream_base="https://api.openai.com", api_key=openai_key), + anthropic=ProviderConfig(upstream_base="https://api.anthropic.com", api_key=anthropic_key), + bedrock_region="us-east-1", + bedrock_upstream_host="bedrock-runtime.us-east-1.amazonaws.com", + redis_host="localhost", + redis_port=6379, + ) + + +@pytest.fixture +def client(): + app = create_app(make_cfg()) + app.testing = True + return app.test_client() + + +def test_healthz_without_params_is_unchanged(client): + resp = client.get("/healthz") + assert resp.status_code == 200 + body = resp.get_json() + assert body == {"status": "ok", "providers": ["anthropic", "bedrock", "openai"]} + + +def test_healthz_model_check_ok(client, monkeypatch): + ok_resp = MagicMock(status_code=200) + monkeypatch.setattr("ventis.llm_proxy.providers.base.requests.get", lambda *a, **k: ok_resp) + + resp = client.get("/healthz?openai_model=gpt-4o-mini") + body = resp.get_json() + assert body["status"] == "ok" + assert body["models"] == {"openai": {"model": "gpt-4o-mini", "ok": True}} + + +def test_healthz_model_check_reports_404_as_degraded(client, monkeypatch): + not_found = MagicMock(status_code=404) + monkeypatch.setattr("ventis.llm_proxy.providers.base.requests.get", lambda *a, **k: not_found) + + resp = client.get("/healthz?anthropic_model=claude-ancient") + body = resp.get_json() + assert body["status"] == "degraded" + assert body["models"]["anthropic"] == { + "model": "claude-ancient", + "ok": False, + "error": "upstream returned 404", + } + # unrequested providers are left alone + assert "openai" not in body["models"] + + +def test_healthz_model_check_without_api_key_fails_fast(): + app = create_app(make_cfg(openai_key=None)) + resp = app.test_client().get("/healthz?openai_model=gpt-4o-mini") + body = resp.get_json() + assert body["status"] == "degraded" + assert body["models"]["openai"] == { + "model": "gpt-4o-mini", + "ok": False, + "error": "no API key configured", + } + + +def _patch_bedrock_clients(monkeypatch, control_client): + """boto3.client("bedrock-runtime", ...) is unused by /healthz; only the + control-plane ("bedrock") client needs a real double.""" + + def fake_client(service_name, **kwargs): + return control_client if service_name == "bedrock" else MagicMock() + + monkeypatch.setattr("ventis.llm_proxy.providers.bedrock.boto3.client", fake_client) + + +def test_bedrock_check_model_ok(monkeypatch): + control = MagicMock(get_foundation_model=MagicMock(return_value={})) + _patch_bedrock_clients(monkeypatch, control) + + app = create_app(make_cfg()) + resp = app.test_client().get("/healthz?bedrock_model=anthropic.claude-3-5-sonnet-20240620-v1:0") + body = resp.get_json() + assert body["models"]["bedrock"]["ok"] is True + assert "invoke access" in body["models"]["bedrock"]["note"] + + +def test_bedrock_check_model_not_found(monkeypatch): + error = ClientError( + {"Error": {"Code": "ResourceNotFoundException", "Message": "model not found"}}, + "GetFoundationModel", + ) + control = MagicMock(get_foundation_model=MagicMock(side_effect=error)) + _patch_bedrock_clients(monkeypatch, control) + + app = create_app(make_cfg()) + resp = app.test_client().get("/healthz?bedrock_model=made-up-model") + body = resp.get_json() + assert body["status"] == "degraded" + assert body["models"]["bedrock"] == { + "model": "made-up-model", + "ok": False, + "error": "model not found", + }