diff --git a/README.md b/README.md index 22a7b2e..9604e2b 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,7 @@ Result: breaking ```bash pip install tool-semantics +# optional marker extras (no provider SDKs): tool-semantics[openai] / [providers] # or from source: pip install -e . ``` diff --git a/docs/probes.md b/docs/probes.md index a999a86..6dd8b14 100644 --- a/docs/probes.md +++ b/docs/probes.md @@ -64,6 +64,76 @@ probe = Probe( | `TOOL_SEMANTICS_MODEL` / `OPENAI_MODEL` | Model id (default `gpt-4o-mini`) | | `TOOL_SEMANTICS_BASE_URL` / `OPENAI_BASE_URL` | OpenAI-compatible base URL | +## Packaging extras (SDK-free) + +Core `pip install tool-semantics` never requires a provider SDK. +`OpenAICompatibleRunner` speaks HTTP with the standard library only. + +Optional extras are **markers** (empty dependency sets) so teams can declare +intent without pulling SDKs: + +```bash +pip install "tool-semantics[openai]" +# or +pip install "tool-semantics[providers]" +``` + +## HTTP provider profile matrix + +| Profile | Base URL pattern | Auth | Notes | +| --- | --- | --- | --- | +| `openai` | `https://api.openai.com/v1` | Bearer | Official Chat Completions | +| `azure-openai` | `https://{resource}.openai.azure.com/openai/deployments/{deployment}` | `api-key` header + `api-version` query | Pass `resource=` / `deployment=` | +| `local` | `http://127.0.0.1:11434/v1` | Bearer (often ignored) | Ollama, vLLM, LM Studio, etc. | + +```python +from tool_semantics.runner import ( + RunnerConfig, + openai_compatible_from_profile, +) + +runner = openai_compatible_from_profile( + "openai", + model="gpt-4o-mini", + api_key="…", +) +azure = openai_compatible_from_profile( + "azure-openai", + model="gpt-4o-mini", # deployment name + api_key="…", + resource="my-resource", + deployment="gpt-4o-mini", +) +local = openai_compatible_from_profile( + "local", + model="llama3.2", + api_key="ollama", # placeholder for local servers that expect a key + base_url="http://127.0.0.1:11434/v1", +) + +cfg = RunnerConfig( + max_retries=2, + retry_backoff_seconds=0.05, + max_calls=20, + max_tokens=50_000, + max_cost_usd=0.50, + prompt_cost_per_1m=0.15, + completion_cost_per_1m=0.60, + max_output_tokens=1024, + seed=1, +) +``` + +Limits (`max_calls`, `max_tokens`, `max_cost_usd`) are enforced on the runner +and recorded on `ModelCompletion.metadata.run_config` together with cumulative +`total_tokens` / `total_cost_usd`. + +### Retry / backoff + +`OpenAICompatibleRunner.complete` attempts `max_retries + 1` times. Between +attempts it sleeps `retry_backoff_seconds` (fixed backoff; default `0.05`). +Unit tests monkeypatch `time.sleep` and never open a live network socket. + ## Safety - Do not embed secrets in probe intents or expected arguments. diff --git a/pyproject.toml b/pyproject.toml index ec64032..b61e90a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,10 @@ Issues = "https://github.com/askmy-stack/tool-semantics/issues" Changelog = "https://github.com/askmy-stack/tool-semantics/blob/main/CHANGELOG.md" [project.optional-dependencies] +# Marker extras — OpenAI / Azure / local profiles use stdlib HTTP only. +# Installing [openai] documents intent; it does not pull a provider SDK. +openai = [] +providers = [] dev = [ "pytest>=8,<10", "pytest-cov>=5,<8", diff --git a/src/tool_semantics/runner.py b/src/tool_semantics/runner.py index 934b33d..9d990d2 100644 --- a/src/tool_semantics/runner.py +++ b/src/tool_semantics/runner.py @@ -3,6 +3,10 @@ Deterministic offline users never need a provider SDK: this module depends only on the standard library plus pydantic (already a core dependency). Provider adapters speak HTTP directly (OpenAI-compatible chat completions). + +Optional extras such as ``tool-semantics[openai]`` are install markers only — +they do not pull SDKs. See ``docs/probes.md`` for the HTTP profile matrix +(OpenAI, Azure OpenAI, local OpenAI-compatible servers). """ from __future__ import annotations @@ -10,8 +14,9 @@ import json import time import urllib.error +import urllib.parse import urllib.request -from typing import Any, Protocol, runtime_checkable +from typing import Any, Literal, Protocol, runtime_checkable from pydantic import BaseModel, Field @@ -21,9 +26,21 @@ class RunnerConfig(BaseModel): timeout_seconds: float = 30.0 max_retries: int = 2 + """Number of retries after the first attempt (total attempts = max_retries + 1).""" + retry_backoff_seconds: float = 0.05 + """Fixed sleep between attempts; tests monkeypatch ``time.sleep``.""" max_calls: int = 50 + """Hard cap on ``complete()`` invocations for this config / runner session.""" + max_tokens: int | None = None + """Optional cumulative prompt+completion token budget across calls.""" + max_cost_usd: float | None = None + """Optional cumulative USD budget when per-1M token rates are set.""" + prompt_cost_per_1m: float | None = None + completion_cost_per_1m: float | None = None temperature: float = 0.0 seed: int | None = None + max_output_tokens: int | None = None + """Forwarded as ``max_tokens`` on the chat-completions request when set.""" extra: dict[str, Any] = Field(default_factory=dict) @@ -48,12 +65,58 @@ class ModelCompletion(BaseModel): text: str | None = None raw: dict[str, Any] = Field(default_factory=dict) metadata: RunnerMetadata + prompt_tokens: int = 0 + completion_tokens: int = 0 + estimated_cost_usd: float = 0.0 class RunnerError(RuntimeError): """Raised when a model runner fails after retries / limits.""" +class HttpProviderProfile(BaseModel): + """Documented generic HTTP profile for OpenAI-compatible endpoints.""" + + name: str + base_url: str + auth_mode: Literal["bearer", "api-key"] = "bearer" + api_key_header: str = "api-key" + query_params: dict[str, str] = Field(default_factory=dict) + provider: str = "openai-compatible" + notes: str = "" + + +# Public profile matrix — no SDKs required. +HTTP_PROVIDER_PROFILES: dict[str, HttpProviderProfile] = { + "openai": HttpProviderProfile( + name="openai", + base_url="https://api.openai.com/v1", + auth_mode="bearer", + provider="openai", + notes="Official OpenAI Chat Completions API.", + ), + "azure-openai": HttpProviderProfile( + name="azure-openai", + base_url=("https://{resource}.openai.azure.com/openai/deployments/{deployment}"), + auth_mode="api-key", + api_key_header="api-key", + query_params={"api-version": "2024-02-15-preview"}, + provider="azure-openai", + notes=( + "Replace {resource} and {deployment}. Path already includes the " + "deployment; requests go to /chat/completions?api-version=…" + ), + ), + "local": HttpProviderProfile( + name="local", + base_url="http://127.0.0.1:11434/v1", + auth_mode="bearer", + provider="local-openai-compatible", + notes="Ollama, vLLM, LM Studio, and other OpenAI-compatible local servers.", + ), +} + + @runtime_checkable class ModelRunner(Protocol): """Separates provider transport from probe evaluation.""" @@ -71,6 +134,65 @@ def complete( ) -> ModelCompletion: ... +def _run_config_dict(cfg: RunnerConfig, **extra: Any) -> dict[str, Any]: + payload: dict[str, Any] = { + "timeout_seconds": cfg.timeout_seconds, + "max_retries": cfg.max_retries, + "retry_backoff_seconds": cfg.retry_backoff_seconds, + "max_calls": cfg.max_calls, + "max_tokens": cfg.max_tokens, + "max_cost_usd": cfg.max_cost_usd, + "prompt_cost_per_1m": cfg.prompt_cost_per_1m, + "completion_cost_per_1m": cfg.completion_cost_per_1m, + "temperature": cfg.temperature, + "seed": cfg.seed, + "max_output_tokens": cfg.max_output_tokens, + **cfg.extra, + **extra, + } + return payload + + +def _usage_from_raw(raw: dict[str, Any]) -> tuple[int, int]: + usage = raw.get("usage") + if not isinstance(usage, dict): + return 0, 0 + prompt = usage.get("prompt_tokens", 0) + completion = usage.get("completion_tokens", 0) + prompt_n = prompt if isinstance(prompt, int) else 0 + completion_n = completion if isinstance(completion, int) else 0 + return prompt_n, completion_n + + +def _estimate_cost_usd( + cfg: RunnerConfig, + *, + prompt_tokens: int, + completion_tokens: int, +) -> float: + cost = 0.0 + if cfg.prompt_cost_per_1m is not None: + cost += (prompt_tokens / 1_000_000.0) * cfg.prompt_cost_per_1m + if cfg.completion_cost_per_1m is not None: + cost += (completion_tokens / 1_000_000.0) * cfg.completion_cost_per_1m + return cost + + +def _enforce_budgets( + *, + call_count: int, + total_tokens: int, + total_cost_usd: float, + cfg: RunnerConfig, +) -> None: + if call_count > cfg.max_calls: + raise RunnerError(f"Exceeded max_calls={cfg.max_calls}") + if cfg.max_tokens is not None and total_tokens > cfg.max_tokens: + raise RunnerError(f"Exceeded max_tokens={cfg.max_tokens} (used {total_tokens})") + if cfg.max_cost_usd is not None and total_cost_usd > cfg.max_cost_usd: + raise RunnerError(f"Exceeded max_cost_usd={cfg.max_cost_usd} (used {total_cost_usd:.6f})") + + class FakeModelRunner: """Deterministic in-memory runner for unit tests (no network).""" @@ -91,6 +213,8 @@ def __init__( run_config={}, ) self.call_count = 0 + self.total_tokens = 0 + self.total_cost_usd = 0.0 @property def metadata(self) -> RunnerMetadata: @@ -107,27 +231,45 @@ def complete( del system, user, tools # unused — scripted responses cfg = config or RunnerConfig() self.call_count += 1 - if self.call_count > cfg.max_calls: - raise RunnerError(f"Exceeded max_calls={cfg.max_calls}") + _enforce_budgets( + call_count=self.call_count, + total_tokens=self.total_tokens, + total_cost_usd=self.total_cost_usd, + cfg=cfg, + ) if self._index >= len(self._responses): raise RunnerError("FakeModelRunner has no remaining scripted responses") completion = self._responses[self._index] self._index += 1 - # Preserve scripted metadata but attach active run config. + prompt_tokens = completion.prompt_tokens + completion_tokens = completion.completion_tokens + if prompt_tokens == 0 and completion_tokens == 0: + prompt_tokens, completion_tokens = _usage_from_raw(completion.raw) + cost = completion.estimated_cost_usd or _estimate_cost_usd( + cfg, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens + ) + self.total_tokens += prompt_tokens + completion_tokens + self.total_cost_usd += cost + _enforce_budgets( + call_count=self.call_count, + total_tokens=self.total_tokens, + total_cost_usd=self.total_cost_usd, + cfg=cfg, + ) return completion.model_copy( update={ + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "estimated_cost_usd": cost, "metadata": completion.metadata.model_copy( update={ - "run_config": { - "timeout_seconds": cfg.timeout_seconds, - "max_retries": cfg.max_retries, - "max_calls": cfg.max_calls, - "temperature": cfg.temperature, - "seed": cfg.seed, - **cfg.extra, - } + "run_config": _run_config_dict( + cfg, + total_tokens=self.total_tokens, + total_cost_usd=self.total_cost_usd, + ) } - ) + ), } ) @@ -143,6 +285,10 @@ def __init__( base_url: str = "https://api.openai.com/v1", provider: str = "openai-compatible", model_version: str | None = None, + auth_mode: Literal["bearer", "api-key"] = "bearer", + api_key_header: str = "api-key", + query_params: dict[str, str] | None = None, + default_headers: dict[str, str] | None = None, ) -> None: if not model: raise ValueError("model is required") @@ -151,11 +297,22 @@ def __init__( self._model = model self._api_key = api_key self._base_url = base_url.rstrip("/") + self._auth_mode = auth_mode + self._api_key_header = api_key_header + self._query_params = dict(query_params or {}) + self._default_headers = dict(default_headers or {}) + self.call_count = 0 + self.total_tokens = 0 + self.total_cost_usd = 0.0 self._metadata = RunnerMetadata( provider=provider, model=model, model_version=model_version, - run_config={"base_url": self._base_url}, + run_config={ + "base_url": self._base_url, + "auth_mode": auth_mode, + "query_params": self._query_params, + }, ) @property @@ -171,6 +328,13 @@ def complete( config: RunnerConfig | None = None, ) -> ModelCompletion: cfg = config or RunnerConfig() + self.call_count += 1 + _enforce_budgets( + call_count=self.call_count, + total_tokens=self.total_tokens, + total_cost_usd=self.total_cost_usd, + cfg=cfg, + ) payload: dict[str, Any] = { "model": self._model, "temperature": cfg.temperature, @@ -183,29 +347,43 @@ def complete( } if cfg.seed is not None: payload["seed"] = cfg.seed + if cfg.max_output_tokens is not None: + payload["max_tokens"] = cfg.max_output_tokens payload.update(cfg.extra) last_error: Exception | None = None attempts = max(1, cfg.max_retries + 1) - for _ in range(attempts): + for attempt in range(attempts): try: raw = self._post(payload, timeout=cfg.timeout_seconds) return self._parse(raw, cfg) except (RunnerError, urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: last_error = exc - time.sleep(0.05) + if attempt + 1 < attempts: + time.sleep(cfg.retry_backoff_seconds) raise RunnerError( f"OpenAI-compatible runner failed after retries: {last_error}" ) from last_error + def _auth_headers(self) -> dict[str, str]: + headers = dict(self._default_headers) + if self._auth_mode == "api-key": + headers[self._api_key_header] = self._api_key + else: + headers["Authorization"] = f"Bearer {self._api_key}" + return headers + def _post(self, payload: dict[str, Any], *, timeout: float) -> dict[str, Any]: body = json.dumps(payload).encode("utf-8") + url = f"{self._base_url}/chat/completions" + if self._query_params: + url = f"{url}?{urllib.parse.urlencode(self._query_params)}" request = urllib.request.Request( - f"{self._base_url}/chat/completions", + url, data=body, headers={ "Content-Type": "application/json", - "Authorization": f"Bearer {self._api_key}", + **self._auth_headers(), }, method="POST", ) @@ -245,29 +423,83 @@ def _parse(self, raw: dict[str, Any], cfg: RunnerConfig) -> ModelCompletion: arguments = {} tool_calls.append(ToolCallRequest(name=name, arguments=arguments)) content = message.get("content") + prompt_tokens, completion_tokens = _usage_from_raw(raw) + cost = _estimate_cost_usd( + cfg, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens + ) + self.total_tokens += prompt_tokens + completion_tokens + self.total_cost_usd += cost + _enforce_budgets( + call_count=self.call_count, + total_tokens=self.total_tokens, + total_cost_usd=self.total_cost_usd, + cfg=cfg, + ) return ModelCompletion( tool_calls=tool_calls, text=content if isinstance(content, str) else None, raw=raw, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + estimated_cost_usd=cost, metadata=RunnerMetadata( provider=self._metadata.provider, model=self._model, model_version=self._metadata.model_version, - run_config={ - "timeout_seconds": cfg.timeout_seconds, - "max_retries": cfg.max_retries, - "max_calls": cfg.max_calls, - "temperature": cfg.temperature, - "seed": cfg.seed, - "base_url": self._base_url, - **cfg.extra, - }, + run_config=_run_config_dict( + cfg, + base_url=self._base_url, + auth_mode=self._auth_mode, + total_tokens=self.total_tokens, + total_cost_usd=self.total_cost_usd, + ), ), ) +def openai_compatible_from_profile( + profile: str | HttpProviderProfile, + *, + model: str, + api_key: str, + base_url: str | None = None, + model_version: str | None = None, + resource: str | None = None, + deployment: str | None = None, +) -> OpenAICompatibleRunner: + """Build a runner from a named or concrete :class:`HttpProviderProfile`.""" + if isinstance(profile, str): + try: + resolved = HTTP_PROVIDER_PROFILES[profile] + except KeyError as exc: + known = ", ".join(sorted(HTTP_PROVIDER_PROFILES)) + raise ValueError(f"Unknown HTTP provider profile {profile!r}; known: {known}") from exc + else: + resolved = profile + url = base_url or resolved.base_url + if "{resource}" in url or "{deployment}" in url: + if not resource or not deployment: + raise ValueError( + "Azure OpenAI profile requires resource= and deployment= " + "(or pass an already-expanded base_url=)" + ) + url = url.format(resource=resource, deployment=deployment) + return OpenAICompatibleRunner( + model=model, + api_key=api_key, + base_url=url, + provider=resolved.provider, + model_version=model_version, + auth_mode=resolved.auth_mode, + api_key_header=resolved.api_key_header, + query_params=resolved.query_params, + ) + + __all__ = [ + "HTTP_PROVIDER_PROFILES", "FakeModelRunner", + "HttpProviderProfile", "ModelCompletion", "ModelRunner", "OpenAICompatibleRunner", @@ -275,4 +507,5 @@ def _parse(self, raw: dict[str, Any], cfg: RunnerConfig) -> ModelCompletion: "RunnerError", "RunnerMetadata", "ToolCallRequest", + "openai_compatible_from_profile", ] diff --git a/tests/test_runner.py b/tests/test_runner.py index 1e35bf3..b8f5ecf 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,6 +1,14 @@ +"""Tests for provider-neutral model runners (no live network).""" + +from __future__ import annotations + +import json +from io import BytesIO + import pytest from tool_semantics.runner import ( + HTTP_PROVIDER_PROFILES, FakeModelRunner, ModelCompletion, OpenAICompatibleRunner, @@ -8,6 +16,7 @@ RunnerError, RunnerMetadata, ToolCallRequest, + openai_compatible_from_profile, ) @@ -30,6 +39,7 @@ def test_fake_runner_records_run_config() -> None: assert completion.metadata.model == "fake-model" assert completion.metadata.run_config["seed"] == 42 assert completion.metadata.run_config["timeout_seconds"] == 1.5 + assert completion.metadata.run_config["retry_backoff_seconds"] == 0.05 def test_fake_runner_respects_max_calls() -> None: @@ -46,6 +56,221 @@ def test_fake_runner_respects_max_calls() -> None: runner.complete(system="s", user="u", tools=[], config=RunnerConfig(max_calls=0)) +def test_fake_runner_respects_max_tokens_and_cost() -> None: + runner = FakeModelRunner( + [ + ModelCompletion( + tool_calls=[], + prompt_tokens=80, + completion_tokens=20, + metadata=RunnerMetadata(provider="fake", model="m"), + ), + ModelCompletion( + tool_calls=[], + prompt_tokens=10, + completion_tokens=10, + metadata=RunnerMetadata(provider="fake", model="m"), + ), + ] + ) + cfg = RunnerConfig( + max_tokens=100, + max_cost_usd=0.001, + prompt_cost_per_1m=5.0, + completion_cost_per_1m=15.0, + ) + first = runner.complete(system="s", user="u", tools=[], config=cfg) + assert first.metadata.run_config["total_tokens"] == 100 + assert first.estimated_cost_usd > 0 + with pytest.raises(RunnerError, match="max_tokens|max_cost_usd"): + runner.complete(system="s", user="u", tools=[], config=cfg) + + def test_openai_compatible_runner_requires_credentials() -> None: with pytest.raises(ValueError, match="api_key"): OpenAICompatibleRunner(model="gpt-test", api_key="") + with pytest.raises(ValueError, match="model"): + OpenAICompatibleRunner(model="", api_key="k") + + +def test_openai_compatible_runner_success_and_usage(monkeypatch: pytest.MonkeyPatch) -> None: + payload = { + "choices": [ + { + "message": { + "content": None, + "tool_calls": [ + { + "function": { + "name": "search_issues", + "arguments": '{"query": "bugs"}', + } + } + ], + } + } + ], + "usage": {"prompt_tokens": 12, "completion_tokens": 3}, + } + captured: dict[str, object] = {} + + class FakeResponse: + def read(self) -> bytes: + return json.dumps(payload).encode("utf-8") + + def __enter__(self) -> FakeResponse: + return self + + def __exit__(self, *args: object) -> None: + return None + + def fake_urlopen(request: object, timeout: float = 0) -> FakeResponse: + captured["timeout"] = timeout + captured["url"] = getattr(request, "full_url", None) + captured["headers"] = dict(getattr(request, "headers", {})) + return FakeResponse() + + monkeypatch.setattr("tool_semantics.runner.urllib.request.urlopen", fake_urlopen) + runner = OpenAICompatibleRunner( + model="gpt-test", + api_key="sk-test", + base_url="http://example/", + ) + completion = runner.complete( + system="sys", + user="find bugs", + tools=[{"type": "function", "function": {"name": "search_issues"}}], + config=RunnerConfig( + seed=9, + max_retries=0, + max_output_tokens=128, + prompt_cost_per_1m=1.0, + completion_cost_per_1m=2.0, + ), + ) + assert completion.tool_calls[0].name == "search_issues" + assert completion.tool_calls[0].arguments == {"query": "bugs"} + assert completion.prompt_tokens == 12 + assert completion.completion_tokens == 3 + assert completion.metadata.run_config["total_tokens"] == 15 + assert completion.metadata.run_config["max_output_tokens"] == 128 + assert "Authorization" in captured["headers"] or "authorization" in { + k.lower() for k in captured["headers"] + } + + +def test_openai_compatible_runner_retries_with_backoff(monkeypatch: pytest.MonkeyPatch) -> None: + import urllib.error + + sleeps: list[float] = [] + calls = {"n": 0} + + def boom(*args: object, **kwargs: object) -> None: + calls["n"] += 1 + raise urllib.error.HTTPError( + "http://example/chat/completions", + 500, + "err", + hdrs=None, # type: ignore[arg-type] + fp=BytesIO(b"upstream failed"), + ) + + monkeypatch.setattr("tool_semantics.runner.urllib.request.urlopen", boom) + monkeypatch.setattr("tool_semantics.runner.time.sleep", lambda s: sleeps.append(s)) + runner = OpenAICompatibleRunner(model="gpt-test", api_key="sk-test") + with pytest.raises(RunnerError, match="failed after retries"): + runner.complete( + system="s", + user="u", + tools=[], + config=RunnerConfig(max_retries=2, retry_backoff_seconds=0.2), + ) + assert calls["n"] == 3 + assert sleeps == [0.2, 0.2] + + +def test_openai_compatible_runner_max_calls(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeResponse: + def read(self) -> bytes: + return json.dumps({"choices": [{"message": {"content": "ok"}}]}).encode() + + def __enter__(self) -> FakeResponse: + return self + + def __exit__(self, *args: object) -> None: + return None + + monkeypatch.setattr( + "tool_semantics.runner.urllib.request.urlopen", + lambda *a, **k: FakeResponse(), + ) + runner = OpenAICompatibleRunner(model="gpt-test", api_key="sk-test") + runner.complete(system="s", user="u", tools=[], config=RunnerConfig(max_calls=1, max_retries=0)) + with pytest.raises(RunnerError, match="max_calls"): + runner.complete( + system="s", user="u", tools=[], config=RunnerConfig(max_calls=1, max_retries=0) + ) + + +def test_http_provider_profiles_and_azure_factory(monkeypatch: pytest.MonkeyPatch) -> None: + assert set(HTTP_PROVIDER_PROFILES) >= {"openai", "azure-openai", "local"} + captured: dict[str, object] = {} + + class FakeResponse: + def read(self) -> bytes: + return json.dumps( + { + "choices": [{"message": {"content": "hi"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ).encode() + + def __enter__(self) -> FakeResponse: + return self + + def __exit__(self, *args: object) -> None: + return None + + def fake_urlopen(request: object, timeout: float = 0) -> FakeResponse: + captured["url"] = getattr(request, "full_url", None) + headers = {str(k).lower(): v for k, v in dict(getattr(request, "headers", {})).items()} + captured["headers"] = headers + return FakeResponse() + + monkeypatch.setattr("tool_semantics.runner.urllib.request.urlopen", fake_urlopen) + runner = openai_compatible_from_profile( + "azure-openai", + model="gpt-4o-mini", + api_key="azure-key", + resource="contoso", + deployment="gpt-4o-mini", + ) + assert runner.metadata.provider == "azure-openai" + runner.complete(system="s", user="u", tools=[], config=RunnerConfig(max_retries=0)) + url = str(captured["url"]) + assert "contoso.openai.azure.com" in url + assert "api-version=" in url + headers = captured["headers"] + assert isinstance(headers, dict) + assert headers.get("api-key") == "azure-key" + + +def test_openai_compatible_from_profile_requires_azure_parts() -> None: + with pytest.raises(ValueError, match="resource"): + openai_compatible_from_profile( + "azure-openai", + model="m", + api_key="k", + ) + with pytest.raises(ValueError, match="Unknown HTTP provider profile"): + openai_compatible_from_profile("nope", model="m", api_key="k") + + +def test_local_profile_factory() -> None: + runner = openai_compatible_from_profile( + "local", + model="llama3.2", + api_key="ollama", + ) + assert runner.metadata.provider == "local-openai-compatible" + assert "11434" in runner.metadata.run_config["base_url"]