CAN-287-Add opt-in per-model check to /healthz

ventis/llm_proxy/README.md

@@ -65,6 +65,39 @@ boto3.client("bedrock-runtime").invoke_model(
                      "messages": [{"role": "user", "content": "hi"}]}))
 ```
 
+## Health checks
+
+`GET /healthz` with no params just reports which provider adapters are
+compiled in — it doesn't call any upstream:
+
+```json
+{"status": "ok", "providers": ["anthropic", "bedrock", "openai"]}
+```
+
+To actually verify a credential can reach a specific model, pass
+`<provider>_model=<id>`:
+
+```bash
+curl 'localhost:8080/healthz?openai_model=gpt-4o-mini&anthropic_model=claude-3-5-sonnet-20241022'
+```
+
+```json
+{
+  "status": "degraded",
+  "providers": ["anthropic", "bedrock", "openai"],
+  "models": {
+    "openai": {"model": "gpt-4o-mini", "ok": true},
+    "anthropic": {"model": "claude-3-5-sonnet-20241022", "ok": false, "error": "upstream returned 404"}
+  }
+}
+```
+
+OpenAI/Anthropic checks are a free `GET /v1/models/{id}` — confirms both the
+key and the model in one request. Bedrock's check (`bedrock_model=<id>`) only
+confirms the model exists in the configured region's catalog; Bedrock has no
+free way to confirm your account actually has invoke access granted for that
+model — that requires a real (billed) `invoke_model` call.
+
 ## Configuration (env vars)
 
 | Var | Default | Purpose |

ventis/llm_proxy/app.py

@@ -27,7 +27,20 @@ def create_app(cfg: Config = None) -> Flask:
 
     @app.route("/healthz", methods=["GET"])
     def healthz():
-        return jsonify(status="ok", providers=sorted(registry.keys()))
+        # Deep model checks are opt-in via `<provider>_model=<id>` query params
+        # so a plain GET /healthz stays a cheap, zero-upstream-call probe.
+        checks = {}
+        for name, prov in registry.items():
+            model_id = request.args.get(f"{name}_model")
+            if model_id:
+                checks[name] = prov.check_model(model_id)
+
+        result = {"status": "ok", "providers": sorted(registry.keys())}
+        if checks:
+            result["models"] = checks
+            if not all(c["ok"] for c in checks.values()):
+                result["status"] = "degraded"
+        return jsonify(**result)
 
     @app.route("/<provider>/<path:subpath>", methods=ALL_METHODS)
     def dispatch(provider, subpath):

ventis/llm_proxy/providers/anthropic.py

@@ -17,3 +17,14 @@ def target(self, req, subpath, body):
             headers=headers,
             params=req.args.to_dict(flat=True),
         )
+
+    def _model_url(self, model_id):
+        return f"{self.cfg.anthropic.upstream_base}/v1/models/{model_id}"
+
+    def _model_check_headers(self):
+        if not self.cfg.anthropic.api_key:
+            return None
+        return {
+            "x-api-key": self.cfg.anthropic.api_key,
+            "anthropic-version": "2023-06-01",
+        }

ventis/llm_proxy/providers/base.py

@@ -10,7 +10,7 @@
 
 import json
 from dataclasses import dataclass, field
-from typing import Dict, Iterable, List, Tuple
+from typing import Dict, Iterable, List, Optional, Tuple
 
 import requests
 
@@ -72,6 +72,14 @@ def __init__(self, cfg):
     def forward(self, req, subpath: str, body: bytes) -> ProxyResponse:
         raise NotImplementedError
 
+    def check_model(self, model_id: str) -> dict:
+        """Verify a specific model is reachable with this provider's credentials.
+
+        Used by ``/healthz?<provider>_model=<id>``. Returns a JSON-serializable
+        dict with at least ``model`` and ``ok``.
+        """
+        raise NotImplementedError
+
 
 class HttpProvider(Provider):
     """Providers that are a straight HTTP reverse-proxy (OpenAI, Anthropic)."""
@@ -94,3 +102,32 @@ def forward(self, req, subpath, body):
             headers=filter_response_headers(resp.headers),
             content=resp.content,
         )
+
+    # -- model existence/access check, shared by OpenAI + Anthropic -------
+    #
+    # Both providers expose a free `GET /v1/models/{id}` that succeeds only if
+    # the credential is valid *and* the model exists/is accessible to the
+    # account, so a real request is cheap enough to make on every check.
+
+    def _model_url(self, model_id: str) -> str:
+        raise NotImplementedError
+
+    def _model_check_headers(self) -> Optional[Dict[str, str]]:
+        """Auth headers for the model check, or None if no key is configured."""
+        raise NotImplementedError
+
+    def check_model(self, model_id: str) -> dict:
+        headers = self._model_check_headers()
+        if headers is None:
+            return {"model": model_id, "ok": False, "error": "no API key configured"}
+        try:
+            resp = requests.get(
+                self._model_url(model_id),
+                headers=headers,
+                timeout=(self.cfg.connect_timeout, self.cfg.read_timeout),
+            )
+        except requests.RequestException as exc:
+            return {"model": model_id, "ok": False, "error": str(exc)}
+        if resp.status_code == 200:
+            return {"model": model_id, "ok": True}
+        return {"model": model_id, "ok": False, "error": f"upstream returned {resp.status_code}"}

ventis/llm_proxy/providers/bedrock.py

@@ -33,6 +33,10 @@ def __init__(self, cfg):
             region_name=cfg.bedrock_region,
             endpoint_url=f"https://{cfg.bedrock_upstream_host}"
         )
+        # Control-plane client (not bedrock-runtime) — only this one exposes
+        # model catalog metadata, and it isn't affected by
+        # AWS_ENDPOINT_URL_BEDROCK_RUNTIME so needs no endpoint override.
+        self._control_client = boto3.client("bedrock", region_name=cfg.bedrock_region)
 
     def forward(self, req, subpath, body):
         model_id, op = self._parse(subpath)
@@ -90,6 +94,27 @@ def forward(self, req, subpath, body):
 
 
 
+    def check_model(self, model_id: str) -> dict:
+        """Confirm ``model_id`` exists in this region's model catalog.
+
+        This only proves the model ID is valid for the configured region — it
+        does NOT confirm this account/role has been granted invoke access,
+        since that's not queryable without an actual (billed) invoke call.
+        """
+        try:
+            self._control_client.get_foundation_model(modelIdentifier=model_id)
+        except ClientError as exc:
+            return {
+                "model": model_id,
+                "ok": False,
+                "error": exc.response.get("Error", {}).get("Message", str(exc)),
+            }
+        return {
+            "model": model_id,
+            "ok": True,
+            "note": "confirms the model exists in this region; does not confirm invoke access is granted",
+        }
+
     @staticmethod
     def _parse(subpath):
         # subpath looks like "model/<modelId>/<op>"; the modelId may itself

ventis/llm_proxy/providers/openai.py

@@ -16,3 +16,11 @@ def target(self, req, subpath, body):
             headers=headers,
             params=req.args.to_dict(flat=True),
         )
+
+    def _model_url(self, model_id):
+        return f"{self.cfg.openai.upstream_base}/v1/models/{model_id}"
+
+    def _model_check_headers(self):
+        if not self.cfg.openai.api_key:
+            return None
+        return {"Authorization": f"Bearer {self.cfg.openai.api_key}"}

ventis/llm_proxy/tests/test_healthz.py

@@ -0,0 +1,118 @@
+"""Tests for /healthz, including the opt-in per-model deep check (CAN-287)."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+from botocore.exceptions import ClientError
+
+from ventis.llm_proxy.app import create_app
+from ventis.llm_proxy.config import Config, ProviderConfig
+
+
+def make_cfg(openai_key="sk-openai", anthropic_key="sk-ant"):
+    return Config(
+        host="127.0.0.1",
+        port=8080,
+        connect_timeout=1.0,
+        read_timeout=1.0,
+        openai=ProviderConfig(upstream_base="https://api.openai.com", api_key=openai_key),
+        anthropic=ProviderConfig(upstream_base="https://api.anthropic.com", api_key=anthropic_key),
+        bedrock_region="us-east-1",
+        bedrock_upstream_host="bedrock-runtime.us-east-1.amazonaws.com",
+        redis_host="localhost",
+        redis_port=6379,
+    )
+
+
+@pytest.fixture
+def client():
+    app = create_app(make_cfg())
+    app.testing = True
+    return app.test_client()
+
+
+def test_healthz_without_params_is_unchanged(client):
+    resp = client.get("/healthz")
+    assert resp.status_code == 200
+    body = resp.get_json()
+    assert body == {"status": "ok", "providers": ["anthropic", "bedrock", "openai"]}
+
+
+def test_healthz_model_check_ok(client, monkeypatch):
+    ok_resp = MagicMock(status_code=200)
+    monkeypatch.setattr("ventis.llm_proxy.providers.base.requests.get", lambda *a, **k: ok_resp)
+
+    resp = client.get("/healthz?openai_model=gpt-4o-mini")
+    body = resp.get_json()
+    assert body["status"] == "ok"
+    assert body["models"] == {"openai": {"model": "gpt-4o-mini", "ok": True}}
+
+
+def test_healthz_model_check_reports_404_as_degraded(client, monkeypatch):
+    not_found = MagicMock(status_code=404)
+    monkeypatch.setattr("ventis.llm_proxy.providers.base.requests.get", lambda *a, **k: not_found)
+
+    resp = client.get("/healthz?anthropic_model=claude-ancient")
+    body = resp.get_json()
+    assert body["status"] == "degraded"
+    assert body["models"]["anthropic"] == {
+        "model": "claude-ancient",
+        "ok": False,
+        "error": "upstream returned 404",
+    }
+    # unrequested providers are left alone
+    assert "openai" not in body["models"]
+
+
+def test_healthz_model_check_without_api_key_fails_fast():
+    app = create_app(make_cfg(openai_key=None))
+    resp = app.test_client().get("/healthz?openai_model=gpt-4o-mini")
+    body = resp.get_json()
+    assert body["status"] == "degraded"
+    assert body["models"]["openai"] == {
+        "model": "gpt-4o-mini",
+        "ok": False,
+        "error": "no API key configured",
+    }
+
+
+def _patch_bedrock_clients(monkeypatch, control_client):
+    """boto3.client("bedrock-runtime", ...) is unused by /healthz; only the
+    control-plane ("bedrock") client needs a real double."""
+
+    def fake_client(service_name, **kwargs):
+        return control_client if service_name == "bedrock" else MagicMock()
+
+    monkeypatch.setattr("ventis.llm_proxy.providers.bedrock.boto3.client", fake_client)
+
+
+def test_bedrock_check_model_ok(monkeypatch):
+    control = MagicMock(get_foundation_model=MagicMock(return_value={}))
+    _patch_bedrock_clients(monkeypatch, control)
+
+    app = create_app(make_cfg())
+    resp = app.test_client().get("/healthz?bedrock_model=anthropic.claude-3-5-sonnet-20240620-v1:0")
+    body = resp.get_json()
+    assert body["models"]["bedrock"]["ok"] is True
+    assert "invoke access" in body["models"]["bedrock"]["note"]
+
+
+def test_bedrock_check_model_not_found(monkeypatch):
+    error = ClientError(
+        {"Error": {"Code": "ResourceNotFoundException", "Message": "model not found"}},
+        "GetFoundationModel",
+    )
+    control = MagicMock(get_foundation_model=MagicMock(side_effect=error))
+    _patch_bedrock_clients(monkeypatch, control)
+
+    app = create_app(make_cfg())
+    resp = app.test_client().get("/healthz?bedrock_model=made-up-model")
+    body = resp.get_json()
+    assert body["status"] == "degraded"
+    assert body["models"]["bedrock"] == {
+        "model": "made-up-model",
+        "ok": False,
+        "error": "model not found",
+    }