From e16083f1aba5656c0a81d92b511edee860200d3f Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Sat, 29 Aug 2026 05:49:31 +0530 Subject: [PATCH 1/3] fix(model): wire openai_compatible configuration across trainer, eval, and config pipeline --- scripts/eval_only.py | 131 ++++++++++- scripts/train.py | 42 +++- skillopt/config.py | 18 ++ skillopt/engine/trainer.py | 21 ++ skillopt/model/__init__.py | 12 + skillopt/model/openai_compatible_backend.py | 18 +- tests/test_openai_compatible_backend.py | 16 ++ tests/test_openai_compatible_config.py | 248 ++++++++++++++++++++ 8 files changed, 494 insertions(+), 12 deletions(-) create mode 100644 tests/test_openai_compatible_config.py diff --git a/scripts/eval_only.py b/scripts/eval_only.py index abe44905..8b6fbb63 100644 --- a/scripts/eval_only.py +++ b/scripts/eval_only.py @@ -38,6 +38,7 @@ configure_copilot_exec, configure_cursor_exec, configure_minimax_chat, + configure_openai_compatible, configure_qwen_chat, set_optimizer_backend, set_optimizer_deployment, @@ -161,7 +162,24 @@ def parse_args() -> argparse.Namespace: # Legacy flat overrides p.add_argument("--env", type=str) p.add_argument("--backend", type=str, - choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "copilot", "copilot_chat", "copilot_exec", "minimax", "minimax_chat"]) + choices=[ + "azure_openai", + "codex", + "codex_exec", + "claude", + "claude_chat", + "claude_code_exec", + "cursor", + "cursor_exec", + "copilot", + "copilot_chat", + "copilot_exec", + "qwen", + "qwen_chat", + "minimax", + "minimax_chat", + "openai_compatible", + ]) p.add_argument("--optimizer_model", type=str) p.add_argument("--target_model", type=str) p.add_argument("--optimizer_backend", type=str) @@ -189,6 +207,45 @@ def parse_args() -> argparse.Namespace: p.add_argument("--target_azure_openai_auth_mode", type=str) p.add_argument("--target_azure_openai_ad_scope", type=str) p.add_argument("--target_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--qwen_chat_base_url", type=str) + p.add_argument("--qwen_chat_api_key", type=str) + p.add_argument("--qwen_chat_temperature", type=float) + p.add_argument("--qwen_chat_timeout_seconds", type=float) + p.add_argument("--qwen_chat_max_tokens", type=int) + p.add_argument("--qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--qwen_chat_thinking_mode", type=str) + p.add_argument("--optimizer_qwen_chat_base_url", type=str) + p.add_argument("--optimizer_qwen_chat_api_key", type=str) + p.add_argument("--optimizer_qwen_chat_temperature", type=float) + p.add_argument("--optimizer_qwen_chat_timeout_seconds", type=float) + p.add_argument("--optimizer_qwen_chat_max_tokens", type=int) + p.add_argument("--optimizer_qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--optimizer_qwen_chat_thinking_mode", type=str) + p.add_argument("--target_qwen_chat_base_url", type=str) + p.add_argument("--target_qwen_chat_api_key", type=str) + p.add_argument("--target_qwen_chat_temperature", type=float) + p.add_argument("--target_qwen_chat_timeout_seconds", type=float) + p.add_argument("--target_qwen_chat_max_tokens", type=int) + p.add_argument("--target_qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--target_qwen_chat_thinking_mode", type=str) + p.add_argument("--openai_compatible_base_url", type=str) + p.add_argument("--openai_compatible_api_key", type=str) + p.add_argument("--openai_compatible_model", type=str) + p.add_argument("--openai_compatible_temperature", type=float) + p.add_argument("--openai_compatible_timeout_seconds", type=float) + p.add_argument("--openai_compatible_max_tokens", type=int) + p.add_argument("--optimizer_openai_compatible_base_url", type=str) + p.add_argument("--optimizer_openai_compatible_api_key", type=str) + p.add_argument("--optimizer_openai_compatible_model", type=str) + p.add_argument("--optimizer_openai_compatible_temperature", type=float) + p.add_argument("--optimizer_openai_compatible_timeout_seconds", type=float) + p.add_argument("--optimizer_openai_compatible_max_tokens", type=int) + p.add_argument("--target_openai_compatible_base_url", type=str) + p.add_argument("--target_openai_compatible_api_key", type=str) + p.add_argument("--target_openai_compatible_model", type=str) + p.add_argument("--target_openai_compatible_temperature", type=float) + p.add_argument("--target_openai_compatible_timeout_seconds", type=float) + p.add_argument("--target_openai_compatible_max_tokens", type=int) p.add_argument("--codex_exec_path", type=str) p.add_argument("--codex_exec_sandbox", type=str) p.add_argument("--codex_exec_profile", type=str) @@ -244,10 +301,8 @@ def parse_args() -> argparse.Namespace: return p.parse_args() -def main() -> None: - args = parse_args() - - from skillopt.config import load_config as _load, flatten_config, is_structured +def load_config(args: argparse.Namespace) -> dict: + from skillopt.config import flatten_config, is_structured, load_config as _load cfg = _load(args.config, overrides=args.cfg_options) structured = is_structured(cfg) @@ -286,6 +341,45 @@ def main() -> None: "target_azure_openai_auth_mode": "model.target_azure_openai_auth_mode", "target_azure_openai_ad_scope": "model.target_azure_openai_ad_scope", "target_azure_openai_managed_identity_client_id": "model.target_azure_openai_managed_identity_client_id", + "qwen_chat_base_url": "model.qwen_chat_base_url", + "qwen_chat_api_key": "model.qwen_chat_api_key", + "qwen_chat_temperature": "model.qwen_chat_temperature", + "qwen_chat_timeout_seconds": "model.qwen_chat_timeout_seconds", + "qwen_chat_max_tokens": "model.qwen_chat_max_tokens", + "qwen_chat_enable_thinking": "model.qwen_chat_enable_thinking", + "qwen_chat_thinking_mode": "model.qwen_chat_thinking_mode", + "optimizer_qwen_chat_base_url": "model.optimizer_qwen_chat_base_url", + "optimizer_qwen_chat_api_key": "model.optimizer_qwen_chat_api_key", + "optimizer_qwen_chat_temperature": "model.optimizer_qwen_chat_temperature", + "optimizer_qwen_chat_timeout_seconds": "model.optimizer_qwen_chat_timeout_seconds", + "optimizer_qwen_chat_max_tokens": "model.optimizer_qwen_chat_max_tokens", + "optimizer_qwen_chat_enable_thinking": "model.optimizer_qwen_chat_enable_thinking", + "optimizer_qwen_chat_thinking_mode": "model.optimizer_qwen_chat_thinking_mode", + "target_qwen_chat_base_url": "model.target_qwen_chat_base_url", + "target_qwen_chat_api_key": "model.target_qwen_chat_api_key", + "target_qwen_chat_temperature": "model.target_qwen_chat_temperature", + "target_qwen_chat_timeout_seconds": "model.target_qwen_chat_timeout_seconds", + "target_qwen_chat_max_tokens": "model.target_qwen_chat_max_tokens", + "target_qwen_chat_enable_thinking": "model.target_qwen_chat_enable_thinking", + "target_qwen_chat_thinking_mode": "model.target_qwen_chat_thinking_mode", + "openai_compatible_base_url": "model.openai_compatible_base_url", + "openai_compatible_api_key": "model.openai_compatible_api_key", + "openai_compatible_model": "model.openai_compatible_model", + "openai_compatible_temperature": "model.openai_compatible_temperature", + "openai_compatible_timeout_seconds": "model.openai_compatible_timeout_seconds", + "openai_compatible_max_tokens": "model.openai_compatible_max_tokens", + "optimizer_openai_compatible_base_url": "model.optimizer_openai_compatible_base_url", + "optimizer_openai_compatible_api_key": "model.optimizer_openai_compatible_api_key", + "optimizer_openai_compatible_model": "model.optimizer_openai_compatible_model", + "optimizer_openai_compatible_temperature": "model.optimizer_openai_compatible_temperature", + "optimizer_openai_compatible_timeout_seconds": "model.optimizer_openai_compatible_timeout_seconds", + "optimizer_openai_compatible_max_tokens": "model.optimizer_openai_compatible_max_tokens", + "target_openai_compatible_base_url": "model.target_openai_compatible_base_url", + "target_openai_compatible_api_key": "model.target_openai_compatible_api_key", + "target_openai_compatible_model": "model.target_openai_compatible_model", + "target_openai_compatible_temperature": "model.target_openai_compatible_temperature", + "target_openai_compatible_timeout_seconds": "model.target_openai_compatible_timeout_seconds", + "target_openai_compatible_max_tokens": "model.target_openai_compatible_max_tokens", "codex_exec_path": "model.codex_exec_path", "codex_exec_sandbox": "model.codex_exec_sandbox", "codex_exec_profile": "model.codex_exec_profile", @@ -466,6 +560,12 @@ def _set_role(key: str, value: str) -> None: cfg["out_root"] = os.path.join("outputs", f"eval_{env}_{model}_{ts}") cfg["out_root"] = os.path.abspath(cfg["out_root"]) + return cfg + + +def main() -> None: + args = parse_args() + cfg = load_config(args) out_root = cfg["out_root"] os.makedirs(out_root, exist_ok=True) @@ -501,6 +601,7 @@ def _set_role(key: str, value: str) -> None: cfg.get("target_azure_openai_managed_identity_client_id") or None ), ) + backend = cfg.get("model_backend") or cfg.get("target_backend") or "azure_openai" set_optimizer_backend(cfg.get("optimizer_backend", "openai_chat")) set_target_backend(cfg.get("target_backend", "openai_chat")) set_optimizer_deployment(cfg.get("optimizer_model", default_model_for_backend(backend))) @@ -554,6 +655,26 @@ def _set_role(key: str, value: str) -> None: minimax_model_cfg = cfg.get("minimax_model") if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat": set_target_deployment(str(minimax_model_cfg)) + configure_openai_compatible( + base_url=cfg.get("openai_compatible_base_url") or None, + api_key=cfg.get("openai_compatible_api_key") or None, + model=cfg.get("openai_compatible_model") or None, + temperature=cfg.get("openai_compatible_temperature"), + timeout_seconds=cfg.get("openai_compatible_timeout_seconds"), + max_tokens=cfg.get("openai_compatible_max_tokens"), + optimizer_base_url=cfg.get("optimizer_openai_compatible_base_url") or None, + optimizer_api_key=cfg.get("optimizer_openai_compatible_api_key") or None, + optimizer_model=cfg.get("optimizer_openai_compatible_model") or None, + optimizer_temperature=cfg.get("optimizer_openai_compatible_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_openai_compatible_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_openai_compatible_max_tokens"), + target_base_url=cfg.get("target_openai_compatible_base_url") or None, + target_api_key=cfg.get("target_openai_compatible_api_key") or None, + target_model=cfg.get("target_openai_compatible_model") or None, + target_temperature=cfg.get("target_openai_compatible_temperature"), + target_timeout_seconds=cfg.get("target_openai_compatible_timeout_seconds"), + target_max_tokens=cfg.get("target_openai_compatible_max_tokens"), + ) set_reasoning_effort(cfg.get("reasoning_effort", "") or None) # Build adapter diff --git a/scripts/train.py b/scripts/train.py index f8f6cbf5..53db7532 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -202,6 +202,24 @@ def parse_args() -> argparse.Namespace: p.add_argument("--minimax_temperature", type=float) p.add_argument("--minimax_max_tokens", type=int) p.add_argument("--minimax_enable_thinking", type=_BOOL) + p.add_argument("--openai_compatible_base_url", type=str) + p.add_argument("--openai_compatible_api_key", type=str) + p.add_argument("--openai_compatible_model", type=str) + p.add_argument("--openai_compatible_temperature", type=float) + p.add_argument("--openai_compatible_timeout_seconds", type=float) + p.add_argument("--openai_compatible_max_tokens", type=int) + p.add_argument("--optimizer_openai_compatible_base_url", type=str) + p.add_argument("--optimizer_openai_compatible_api_key", type=str) + p.add_argument("--optimizer_openai_compatible_model", type=str) + p.add_argument("--optimizer_openai_compatible_temperature", type=float) + p.add_argument("--optimizer_openai_compatible_timeout_seconds", type=float) + p.add_argument("--optimizer_openai_compatible_max_tokens", type=int) + p.add_argument("--target_openai_compatible_base_url", type=str) + p.add_argument("--target_openai_compatible_api_key", type=str) + p.add_argument("--target_openai_compatible_model", type=str) + p.add_argument("--target_openai_compatible_temperature", type=float) + p.add_argument("--target_openai_compatible_timeout_seconds", type=float) + p.add_argument("--target_openai_compatible_max_tokens", type=int) p.add_argument("--codex_exec_path", type=str) p.add_argument("--codex_exec_sandbox", type=str) p.add_argument("--codex_exec_profile", type=str) @@ -426,6 +444,24 @@ def _retired_option_sources( "minimax_temperature": "model.minimax_temperature", "minimax_max_tokens": "model.minimax_max_tokens", "minimax_enable_thinking": "model.minimax_enable_thinking", + "openai_compatible_base_url": "model.openai_compatible_base_url", + "openai_compatible_api_key": "model.openai_compatible_api_key", + "openai_compatible_model": "model.openai_compatible_model", + "openai_compatible_temperature": "model.openai_compatible_temperature", + "openai_compatible_timeout_seconds": "model.openai_compatible_timeout_seconds", + "openai_compatible_max_tokens": "model.openai_compatible_max_tokens", + "optimizer_openai_compatible_base_url": "model.optimizer_openai_compatible_base_url", + "optimizer_openai_compatible_api_key": "model.optimizer_openai_compatible_api_key", + "optimizer_openai_compatible_model": "model.optimizer_openai_compatible_model", + "optimizer_openai_compatible_temperature": "model.optimizer_openai_compatible_temperature", + "optimizer_openai_compatible_timeout_seconds": "model.optimizer_openai_compatible_timeout_seconds", + "optimizer_openai_compatible_max_tokens": "model.optimizer_openai_compatible_max_tokens", + "target_openai_compatible_base_url": "model.target_openai_compatible_base_url", + "target_openai_compatible_api_key": "model.target_openai_compatible_api_key", + "target_openai_compatible_model": "model.target_openai_compatible_model", + "target_openai_compatible_temperature": "model.target_openai_compatible_temperature", + "target_openai_compatible_timeout_seconds": "model.target_openai_compatible_timeout_seconds", + "target_openai_compatible_max_tokens": "model.target_openai_compatible_max_tokens", "codex_exec_path": "model.codex_exec_path", "codex_exec_sandbox": "model.codex_exec_sandbox", "codex_exec_profile": "model.codex_exec_profile", @@ -484,7 +520,8 @@ def _retired_option_sources( def load_config(args: argparse.Namespace) -> dict: """Load config with _base_ inheritance, then apply CLI overrides.""" import warnings - from skillopt.config import load_config as _load, flatten_config, is_structured + + from skillopt.config import flatten_config, is_structured, load_config as _load # F08: Warn when API keys are supplied on the CLI. Keep the replacement # guidance specific to each backend and, where applicable, each role. @@ -509,6 +546,9 @@ def load_config(args: argparse.Namespace) -> dict: "optimizer_qwen_chat_api_key": "OPTIMIZER_QWEN_CHAT_API_KEY", "target_qwen_chat_api_key": "TARGET_QWEN_CHAT_API_KEY", "minimax_api_key": "MINIMAX_API_KEY", + "openai_compatible_api_key": "OPENAI_COMPATIBLE_API_KEY", + "optimizer_openai_compatible_api_key": "OPTIMIZER_OPENAI_COMPATIBLE_API_KEY", + "target_openai_compatible_api_key": "TARGET_OPENAI_COMPATIBLE_API_KEY", } for _cli_key, _guidance in _credential_guidance.items(): if getattr(args, _cli_key, None): diff --git a/skillopt/config.py b/skillopt/config.py index e1fdec04..12230d31 100644 --- a/skillopt/config.py +++ b/skillopt/config.py @@ -123,6 +123,24 @@ "model.minimax_temperature": "minimax_temperature", "model.minimax_max_tokens": "minimax_max_tokens", "model.minimax_enable_thinking": "minimax_enable_thinking", + "model.openai_compatible_base_url": "openai_compatible_base_url", + "model.openai_compatible_api_key": "openai_compatible_api_key", + "model.openai_compatible_model": "openai_compatible_model", + "model.openai_compatible_temperature": "openai_compatible_temperature", + "model.openai_compatible_timeout_seconds": "openai_compatible_timeout_seconds", + "model.openai_compatible_max_tokens": "openai_compatible_max_tokens", + "model.optimizer_openai_compatible_base_url": "optimizer_openai_compatible_base_url", + "model.optimizer_openai_compatible_api_key": "optimizer_openai_compatible_api_key", + "model.optimizer_openai_compatible_model": "optimizer_openai_compatible_model", + "model.optimizer_openai_compatible_temperature": "optimizer_openai_compatible_temperature", + "model.optimizer_openai_compatible_timeout_seconds": "optimizer_openai_compatible_timeout_seconds", + "model.optimizer_openai_compatible_max_tokens": "optimizer_openai_compatible_max_tokens", + "model.target_openai_compatible_base_url": "target_openai_compatible_base_url", + "model.target_openai_compatible_api_key": "target_openai_compatible_api_key", + "model.target_openai_compatible_model": "target_openai_compatible_model", + "model.target_openai_compatible_temperature": "target_openai_compatible_temperature", + "model.target_openai_compatible_timeout_seconds": "target_openai_compatible_timeout_seconds", + "model.target_openai_compatible_max_tokens": "target_openai_compatible_max_tokens", "train.num_epochs": "num_epochs", "train.train_size": "train_size", "train.steps_per_epoch": "steps_per_epoch", diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index 520a90eb..d461345a 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -67,6 +67,7 @@ configure_copilot_exec, configure_cursor_exec, configure_minimax_chat, + configure_openai_compatible, configure_qwen_chat, get_qwen_thinking_modes, get_token_summary, @@ -832,6 +833,26 @@ def _build_eval_env(split: str, env_num: int, seed: int): minimax_model_cfg = cfg.get("minimax_model") if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat": set_target_deployment(str(minimax_model_cfg)) + configure_openai_compatible( + base_url=cfg.get("openai_compatible_base_url") or None, + api_key=cfg.get("openai_compatible_api_key") or None, + model=cfg.get("openai_compatible_model") or None, + temperature=cfg.get("openai_compatible_temperature"), + timeout_seconds=cfg.get("openai_compatible_timeout_seconds"), + max_tokens=cfg.get("openai_compatible_max_tokens"), + optimizer_base_url=cfg.get("optimizer_openai_compatible_base_url") or None, + optimizer_api_key=cfg.get("optimizer_openai_compatible_api_key") or None, + optimizer_model=cfg.get("optimizer_openai_compatible_model") or None, + optimizer_temperature=cfg.get("optimizer_openai_compatible_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_openai_compatible_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_openai_compatible_max_tokens"), + target_base_url=cfg.get("target_openai_compatible_base_url") or None, + target_api_key=cfg.get("target_openai_compatible_api_key") or None, + target_model=cfg.get("target_openai_compatible_model") or None, + target_temperature=cfg.get("target_openai_compatible_temperature"), + target_timeout_seconds=cfg.get("target_openai_compatible_timeout_seconds"), + target_max_tokens=cfg.get("target_openai_compatible_max_tokens"), + ) _configure_trace_to_optimizer_gates(target_backend, cfg) reasoning = cfg.get("reasoning_effort", "") or None set_reasoning_effort(reasoning) diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py index 9033bc98..1d0edd02 100644 --- a/skillopt/model/__init__.py +++ b/skillopt/model/__init__.py @@ -764,9 +764,15 @@ def configure_openai_compatible( optimizer_base_url: str | None = None, optimizer_api_key: str | None = None, optimizer_model: str | None = None, + optimizer_temperature: float | str | None = None, + optimizer_timeout_seconds: float | str | None = None, + optimizer_max_tokens: int | str | None = None, target_base_url: str | None = None, target_api_key: str | None = None, target_model: str | None = None, + target_temperature: float | str | None = None, + target_timeout_seconds: float | str | None = None, + target_max_tokens: int | str | None = None, ) -> None: _openai_compat.configure_openai_compatible( base_url=base_url, @@ -778,9 +784,15 @@ def configure_openai_compatible( optimizer_base_url=optimizer_base_url, optimizer_api_key=optimizer_api_key, optimizer_model=optimizer_model, + optimizer_temperature=optimizer_temperature, + optimizer_timeout_seconds=optimizer_timeout_seconds, + optimizer_max_tokens=optimizer_max_tokens, target_base_url=target_base_url, target_api_key=target_api_key, target_model=target_model, + target_temperature=target_temperature, + target_timeout_seconds=target_timeout_seconds, + target_max_tokens=target_max_tokens, ) diff --git a/skillopt/model/openai_compatible_backend.py b/skillopt/model/openai_compatible_backend.py index 7462be41..4fef4ce1 100644 --- a/skillopt/model/openai_compatible_backend.py +++ b/skillopt/model/openai_compatible_backend.py @@ -372,9 +372,15 @@ def configure_openai_compatible( optimizer_base_url: str | None = None, optimizer_api_key: str | None = None, optimizer_model: str | None = None, + optimizer_temperature: float | str | None = None, + optimizer_timeout_seconds: float | str | None = None, + optimizer_max_tokens: int | str | None = None, target_base_url: str | None = None, target_api_key: str | None = None, target_model: str | None = None, + target_temperature: float | str | None = None, + target_timeout_seconds: float | str | None = None, + target_max_tokens: int | str | None = None, ) -> None: """Configure the generic OpenAI-compatible backend at runtime. @@ -400,9 +406,9 @@ def configure_openai_compatible( base_url=optimizer_base_url if optimizer_base_url is not None else base_url, api_key=optimizer_api_key if optimizer_api_key is not None else api_key, deployment=optimizer_model if optimizer_model is not None else model, - temperature=temperature, - timeout_seconds=timeout_seconds, - max_tokens=max_tokens, + temperature=optimizer_temperature if optimizer_temperature is not None else temperature, + timeout_seconds=optimizer_timeout_seconds if optimizer_timeout_seconds is not None else timeout_seconds, + max_tokens=optimizer_max_tokens if optimizer_max_tokens is not None else max_tokens, ) _update_config( TARGET_CONFIG, @@ -410,9 +416,9 @@ def configure_openai_compatible( base_url=target_base_url if target_base_url is not None else base_url, api_key=target_api_key if target_api_key is not None else api_key, deployment=target_model if target_model is not None else model, - temperature=temperature, - timeout_seconds=timeout_seconds, - max_tokens=max_tokens, + temperature=target_temperature if target_temperature is not None else temperature, + timeout_seconds=target_timeout_seconds if target_timeout_seconds is not None else timeout_seconds, + max_tokens=target_max_tokens if target_max_tokens is not None else max_tokens, ) _reset_clients() diff --git a/tests/test_openai_compatible_backend.py b/tests/test_openai_compatible_backend.py index 4482e206..f5abe41b 100644 --- a/tests/test_openai_compatible_backend.py +++ b/tests/test_openai_compatible_backend.py @@ -49,20 +49,36 @@ def test_configure_preserves_role_specific_values() -> None: base_url="https://shared.example/v1", api_key="shared-key", model="shared-model", + temperature=0.5, + timeout_seconds=60, + max_tokens=2048, optimizer_base_url="https://optimizer.example/v1", optimizer_api_key="optimizer-key", optimizer_model="optimizer-model", + optimizer_temperature=0.2, + optimizer_timeout_seconds=120, + optimizer_max_tokens=4096, target_base_url="https://target.example/v1", target_api_key="target-key", target_model="target-model", + target_temperature=0.7, + target_timeout_seconds=90, + target_max_tokens=8192, ) assert backend.OPTIMIZER_CONFIG.base_url == "https://optimizer.example/v1" assert backend.OPTIMIZER_CONFIG.api_key == "optimizer-key" assert backend.OPTIMIZER_CONFIG.deployment == "optimizer-model" + assert backend.OPTIMIZER_CONFIG.temperature == 0.2 + assert backend.OPTIMIZER_CONFIG.timeout_seconds == 120.0 + assert backend.OPTIMIZER_CONFIG.max_tokens == 4096 + assert backend.TARGET_CONFIG.base_url == "https://target.example/v1" assert backend.TARGET_CONFIG.api_key == "target-key" assert backend.TARGET_CONFIG.deployment == "target-model" + assert backend.TARGET_CONFIG.temperature == 0.7 + assert backend.TARGET_CONFIG.timeout_seconds == 90.0 + assert backend.TARGET_CONFIG.max_tokens == 8192 def test_optimizer_and_target_route_to_their_own_clients(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_openai_compatible_config.py b/tests/test_openai_compatible_config.py new file mode 100644 index 00000000..da464c13 --- /dev/null +++ b/tests/test_openai_compatible_config.py @@ -0,0 +1,248 @@ +"""Tests for OpenAI-compatible backend configuration pipeline.""" + +from __future__ import annotations + +import tempfile +from unittest import mock + +import pytest +import yaml + +import scripts.eval_only as eval_only_script +import scripts.train as train_script +from skillopt.config import flatten_config, load_config + + +def test_flatten_config_maps_openai_compatible_keys() -> None: + structured_cfg = { + "model": { + "backend": "openai_compatible", + "openai_compatible_base_url": "https://api.deepseek.com/v1", + "openai_compatible_api_key": "sk-deepseek-test", + "openai_compatible_model": "deepseek-chat", + "openai_compatible_temperature": 0.3, + "openai_compatible_timeout_seconds": 180.0, + "openai_compatible_max_tokens": 4096, + "optimizer_openai_compatible_base_url": "https://api.together.xyz/v1", + "optimizer_openai_compatible_api_key": "sk-together-test", + "optimizer_openai_compatible_model": "together-model", + "optimizer_openai_compatible_temperature": 0.1, + "optimizer_openai_compatible_timeout_seconds": 240.0, + "optimizer_openai_compatible_max_tokens": 8192, + "target_openai_compatible_base_url": "https://api.groq.com/openai/v1", + "target_openai_compatible_api_key": "sk-groq-test", + "target_openai_compatible_model": "groq-model", + "target_openai_compatible_temperature": 0.7, + "target_openai_compatible_timeout_seconds": 90.0, + "target_openai_compatible_max_tokens": 2048, + }, + "train": {"num_epochs": 1}, + "env": {"name": "searchqa"}, + } + + flat = flatten_config(structured_cfg) + + assert flat["model_backend"] == "openai_compatible" + assert flat["openai_compatible_base_url"] == "https://api.deepseek.com/v1" + assert flat["openai_compatible_api_key"] == "sk-deepseek-test" + assert flat["openai_compatible_model"] == "deepseek-chat" + assert flat["openai_compatible_temperature"] == 0.3 + assert flat["openai_compatible_timeout_seconds"] == 180.0 + assert flat["openai_compatible_max_tokens"] == 4096 + + assert flat["optimizer_openai_compatible_base_url"] == "https://api.together.xyz/v1" + assert flat["optimizer_openai_compatible_api_key"] == "sk-together-test" + assert flat["optimizer_openai_compatible_model"] == "together-model" + assert flat["optimizer_openai_compatible_temperature"] == 0.1 + assert flat["optimizer_openai_compatible_timeout_seconds"] == 240.0 + assert flat["optimizer_openai_compatible_max_tokens"] == 8192 + + assert flat["target_openai_compatible_base_url"] == "https://api.groq.com/openai/v1" + assert flat["target_openai_compatible_api_key"] == "sk-groq-test" + assert flat["target_openai_compatible_model"] == "groq-model" + assert flat["target_openai_compatible_temperature"] == 0.7 + assert flat["target_openai_compatible_timeout_seconds"] == 90.0 + assert flat["target_openai_compatible_max_tokens"] == 2048 + + +def test_load_config_with_cfg_options_overrides() -> None: + raw = { + "model": { + "backend": "openai_compatible", + "openai_compatible_base_url": "http://localhost:11434/v1", + "openai_compatible_model": "llama3", + }, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + path = f.name + + overrides = [ + "model.openai_compatible_base_url=http://custom-host:8000/v1", + "model.openai_compatible_temperature=0.8", + ] + cfg = load_config(path, overrides=overrides) + flat = flatten_config(cfg) + + assert flat["openai_compatible_base_url"] == "http://custom-host:8000/v1" + assert flat["openai_compatible_temperature"] == 0.8 + assert flat["openai_compatible_model"] == "llama3" + + +def test_train_script_cli_to_structured_mapping() -> None: + cli_args = [ + "--config", "configs/base.yaml", + "--openai_compatible_base_url", "https://api.openai-compat.com/v1", + "--openai_compatible_model", "custom-model", + "--openai_compatible_temperature", "0.4", + "--openai_compatible_timeout_seconds", "150", + "--openai_compatible_max_tokens", "3000", + "--optimizer_openai_compatible_base_url", "https://api.opt.com/v1", + "--target_openai_compatible_base_url", "https://api.target.com/v1", + ] + + with mock.patch("sys.argv", ["train.py"] + cli_args): + args = train_script.parse_args() + + assert args.openai_compatible_base_url == "https://api.openai-compat.com/v1" + assert args.openai_compatible_model == "custom-model" + assert args.openai_compatible_temperature == 0.4 + assert args.openai_compatible_timeout_seconds == 150.0 + assert args.openai_compatible_max_tokens == 3000 + assert args.optimizer_openai_compatible_base_url == "https://api.opt.com/v1" + assert args.target_openai_compatible_base_url == "https://api.target.com/v1" + + for flag in [ + "openai_compatible_base_url", + "openai_compatible_api_key", + "openai_compatible_model", + "openai_compatible_temperature", + "openai_compatible_timeout_seconds", + "openai_compatible_max_tokens", + "optimizer_openai_compatible_base_url", + "optimizer_openai_compatible_api_key", + "optimizer_openai_compatible_model", + "optimizer_openai_compatible_temperature", + "optimizer_openai_compatible_timeout_seconds", + "optimizer_openai_compatible_max_tokens", + "target_openai_compatible_base_url", + "target_openai_compatible_api_key", + "target_openai_compatible_model", + "target_openai_compatible_temperature", + "target_openai_compatible_timeout_seconds", + "target_openai_compatible_max_tokens", + ]: + assert flag in train_script._LEGACY_TO_STRUCTURED + assert train_script._LEGACY_TO_STRUCTURED[flag] == f"model.{flag}" + + +def test_eval_only_parse_args_and_map() -> None: + raw = { + "model": { + "backend": "openai_compatible", + }, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + cli_args = [ + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + "--openai_compatible_base_url", "https://eval.example/v1", + "--openai_compatible_model", "eval-model", + "--target_openai_compatible_temperature", "0.6", + ] + + with mock.patch("sys.argv", ["eval_only.py"] + cli_args): + args = eval_only_script.parse_args() + cfg = eval_only_script.load_config(args) + + assert args.backend == "openai_compatible" + assert args.openai_compatible_base_url == "https://eval.example/v1" + assert args.openai_compatible_model == "eval-model" + assert args.target_openai_compatible_temperature == 0.6 + assert cfg["openai_compatible_base_url"] == "https://eval.example/v1" + assert cfg["openai_compatible_model"] == "eval-model" + assert cfg["target_openai_compatible_temperature"] == 0.6 + + +def test_train_script_credential_warnings_for_openai_compatible() -> None: + raw = { + "model": {"backend": "openai_compatible"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + cli_args = [ + "--config", config_path, + "--openai_compatible_api_key", "sk-secret-test", + ] + + with mock.patch("sys.argv", ["train.py"] + cli_args): + with pytest.deprecated_call(match="OPENAI_COMPATIBLE_API_KEY"): + train_script.load_config(train_script.parse_args()) + + +def test_trainer_initialization_configures_openai_compatible(monkeypatch: pytest.MonkeyPatch) -> None: + import skillopt.engine.trainer as trainer_mod + from skillopt.envs.base import EnvAdapter + + configured_kwargs: dict = {} + + def fake_configure(**kwargs): + configured_kwargs.update(kwargs) + + monkeypatch.setattr(trainer_mod, "configure_openai_compatible", fake_configure) + + class EarlyExit(Exception): + pass + + def stop_after_model_config(*args, **kwargs): + raise EarlyExit() + + monkeypatch.setattr(trainer_mod, "_configure_trace_to_optimizer_gates", stop_after_model_config) + + cfg = { + "model_backend": "openai_compatible", + "optimizer_backend": "openai_compatible", + "target_backend": "openai_compatible", + "optimizer_model": "opt-model", + "target_model": "target-model", + "openai_compatible_base_url": "https://api.test.com/v1", + "openai_compatible_api_key": "test-key", + "openai_compatible_model": "test-model", + "openai_compatible_temperature": 0.5, + "openai_compatible_timeout_seconds": 120.0, + "openai_compatible_max_tokens": 4096, + "skill_init": "skills/empty.md", + "num_epochs": 1, + "train_size": 1, + "batch_size": 1, + "accumulation": 1, + "merge_batch_size": 2, + "edit_budget": 2, + "seed": 42, + "out_root": "/tmp/out", + } + + mock_adapter = mock.create_autospec(EnvAdapter, instance=True) + mock_adapter.requires_ray.return_value = False + mock_adapter.get_dataloader.return_value = None + + trainer = trainer_mod.ReflACTTrainer(cfg, mock_adapter) + with pytest.raises(EarlyExit): + trainer.train() + + assert configured_kwargs["base_url"] == "https://api.test.com/v1" + assert configured_kwargs["api_key"] == "test-key" + assert configured_kwargs["model"] == "test-model" + assert configured_kwargs["temperature"] == 0.5 + assert configured_kwargs["timeout_seconds"] == 120.0 + assert configured_kwargs["max_tokens"] == 4096 + From 451f2e8f7b7ffe7fd34d0466b3d345199b2dcfbf Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Sun, 30 Aug 2026 23:33:49 +0530 Subject: [PATCH 2/3] fix(model): resolve entry-point gaps in train and eval-only pipelines --- scripts/eval_only.py | 42 +++++ scripts/train.py | 22 ++- tests/test_openai_compatible_config.py | 212 +++++++++++++++++++++++++ 3 files changed, 275 insertions(+), 1 deletion(-) diff --git a/scripts/eval_only.py b/scripts/eval_only.py index 8b6fbb63..46508cc5 100644 --- a/scripts/eval_only.py +++ b/scripts/eval_only.py @@ -493,6 +493,9 @@ def _set_role(key: str, value: str) -> None: elif backend == "copilot_exec": _set_role("optimizer_backend", "openai_chat") _set_role("target_backend", "copilot_exec") + elif backend == "qwen_chat": + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "qwen_chat") elif backend == "minimax_chat": _set_role("optimizer_backend", "openai_chat") _set_role("target_backend", "minimax_chat") @@ -518,6 +521,22 @@ def _set_role(key: str, value: str) -> None: and not _has_model_override("model.optimizer", "optimizer_model") ): cfg["optimizer_model"] = default_model_for_backend("claude_code_exec") + if cfg.get("optimizer_backend") == "qwen_chat": + if ( + str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + cfg["optimizer_model"] = default_model_for_backend("qwen_chat") + if cfg.get("optimizer_backend") == "openai_compatible": + if ( + str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + cfg["optimizer_model"] = ( + cfg.get("optimizer_openai_compatible_model") + or cfg.get("openai_compatible_model") + or default_model_for_backend("openai_compatible") + ) if cfg.get("target_backend") == "claude_chat": if ( str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS @@ -543,6 +562,12 @@ def _set_role(key: str, value: str) -> None: ): # Copilot CLI model IDs are independent of Azure deployment names. cfg["target_model"] = "" + if cfg.get("target_backend") == "qwen_chat": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = default_model_for_backend("qwen_chat") if cfg.get("target_backend") == "minimax_chat": if ( str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS @@ -552,6 +577,16 @@ def _set_role(key: str, value: str) -> None: cfg.get("minimax_model") or default_model_for_backend("minimax_chat") ) + if cfg.get("target_backend") == "openai_compatible": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = ( + cfg.get("target_openai_compatible_model") + or cfg.get("openai_compatible_model") + or default_model_for_backend("openai_compatible") + ) if not cfg.get("out_root"): env = cfg.get("env", "unknown") @@ -636,6 +671,13 @@ def main() -> None: max_tokens=cfg.get("qwen_chat_max_tokens"), enable_thinking=cfg.get("qwen_chat_enable_thinking"), thinking_mode=cfg.get("qwen_chat_thinking_mode"), + optimizer_base_url=cfg.get("optimizer_qwen_chat_base_url") or None, + optimizer_api_key=cfg.get("optimizer_qwen_chat_api_key") or None, + optimizer_temperature=cfg.get("optimizer_qwen_chat_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_qwen_chat_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_qwen_chat_max_tokens"), + optimizer_enable_thinking=cfg.get("optimizer_qwen_chat_enable_thinking"), + optimizer_thinking_mode=cfg.get("optimizer_qwen_chat_thinking_mode"), target_base_url=cfg.get("target_qwen_chat_base_url") or None, target_api_key=cfg.get("target_qwen_chat_api_key") or None, target_temperature=cfg.get("target_qwen_chat_temperature"), diff --git a/scripts/train.py b/scripts/train.py index 53db7532..af162659 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -144,7 +144,7 @@ def parse_args() -> argparse.Namespace: # Legacy flat CLI overrides (still work, prefer --cfg-options for new usage) p.add_argument("--env", type=str) p.add_argument("--backend", type=str, - choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "copilot", "copilot_chat", "copilot_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"]) + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "copilot", "copilot_chat", "copilot_exec", "qwen", "qwen_chat", "minimax", "minimax_chat", "openai_compatible"]) p.add_argument("--optimizer_model", type=str) p.add_argument("--target_model", type=str) p.add_argument("--optimizer_backend", type=str) @@ -744,6 +744,16 @@ def _set_role(key: str, value: str) -> None: and not _has_model_override("model.optimizer", "optimizer_model") ): flat["optimizer_model"] = default_model_for_backend("qwen_chat") + if flat.get("optimizer_backend") == "openai_compatible": + if ( + str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + flat["optimizer_model"] = ( + flat.get("optimizer_openai_compatible_model") + or flat.get("openai_compatible_model") + or default_model_for_backend("openai_compatible") + ) if flat.get("target_backend") == "claude_chat": if ( str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS @@ -784,6 +794,16 @@ def _set_role(key: str, value: str) -> None: flat.get("minimax_model") or default_model_for_backend("minimax_chat") ) + if flat.get("target_backend") == "openai_compatible": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = ( + flat.get("target_openai_compatible_model") + or flat.get("openai_compatible_model") + or default_model_for_backend("openai_compatible") + ) # Auto-generate output root if not flat.get("out_root"): diff --git a/tests/test_openai_compatible_config.py b/tests/test_openai_compatible_config.py index da464c13..bec6e709 100644 --- a/tests/test_openai_compatible_config.py +++ b/tests/test_openai_compatible_config.py @@ -246,3 +246,215 @@ def stop_after_model_config(*args, **kwargs): assert configured_kwargs["timeout_seconds"] == 120.0 assert configured_kwargs["max_tokens"] == 4096 + +def test_train_script_backend_choices_accepts_openai_compatible() -> None: + raw = { + "model": {"backend": "azure_openai", "optimizer": "gpt-5.5", "target": "gpt-5.5"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + cli_args = [ + "--config", config_path, + "--backend", "openai_compatible", + ] + with mock.patch("sys.argv", ["train.py"] + cli_args): + args = train_script.parse_args() + flat = train_script.load_config(args) + + assert args.backend == "openai_compatible" + assert flat["optimizer_backend"] == "openai_compatible" + assert flat["target_backend"] == "openai_compatible" + assert flat["optimizer_model"] == "gpt-4o-mini" + assert flat["target_model"] == "gpt-4o-mini" + + +def test_train_script_openai_compatible_model_precedence() -> None: + raw = { + "model": {"backend": "azure_openai", "optimizer": "gpt-5.5", "target": "gpt-5.5"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + # 1. Shared model override + with mock.patch("sys.argv", [ + "train.py", + "--config", config_path, + "--backend", "openai_compatible", + "--openai_compatible_model", "deepseek-chat", + ]): + flat = train_script.load_config(train_script.parse_args()) + assert flat["optimizer_model"] == "deepseek-chat" + assert flat["target_model"] == "deepseek-chat" + + # 2. Per-role compatible model overrides + with mock.patch("sys.argv", [ + "train.py", + "--config", config_path, + "--backend", "openai_compatible", + "--openai_compatible_model", "fallback-shared", + "--optimizer_openai_compatible_model", "deepseek-coder", + "--target_openai_compatible_model", "deepseek-v3", + ]): + flat = train_script.load_config(train_script.parse_args()) + assert flat["optimizer_model"] == "deepseek-coder" + assert flat["target_model"] == "deepseek-v3" + + # 3. Explicit per-role model overrides take highest precedence + with mock.patch("sys.argv", [ + "train.py", + "--config", config_path, + "--backend", "openai_compatible", + "--openai_compatible_model", "fallback-shared", + "--optimizer_model", "explicit-opt-model", + "--target_model", "explicit-tgt-model", + ]): + flat = train_script.load_config(train_script.parse_args()) + assert flat["optimizer_model"] == "explicit-opt-model" + assert flat["target_model"] == "explicit-tgt-model" + + +@pytest.mark.parametrize("backend_flag", ["qwen", "qwen_chat"]) +def test_eval_only_qwen_role_and_model_resolution(backend_flag: str) -> None: + raw = { + "model": {"backend": "azure_openai", "optimizer": "gpt-5.5", "target": "gpt-5.5"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + cli_args = [ + "--config", config_path, + "--skill", "skills/test.md", + "--backend", backend_flag, + ] + with mock.patch("sys.argv", ["eval_only.py"] + cli_args): + args = eval_only_script.parse_args() + cfg = eval_only_script.load_config(args) + + assert cfg["optimizer_backend"] == "openai_chat" + assert cfg["target_backend"] == "qwen_chat" + assert cfg["optimizer_model"] == "gpt-5.5" # openai_chat keeps base sentinel or default + assert cfg["target_model"] == "Qwen/Qwen3.5-4B" # normalized from gpt-5.5 sentinel + + +def test_eval_only_openai_compatible_model_precedence() -> None: + raw = { + "model": {"backend": "azure_openai", "optimizer": "gpt-5.5", "target": "gpt-5.5"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + # 1. Default fallback + with mock.patch("sys.argv", [ + "eval_only.py", + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + ]): + cfg = eval_only_script.load_config(eval_only_script.parse_args()) + assert cfg["optimizer_model"] == "gpt-4o-mini" + assert cfg["target_model"] == "gpt-4o-mini" + + # 2. Shared model override + with mock.patch("sys.argv", [ + "eval_only.py", + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + "--openai_compatible_model", "deepseek-chat", + ]): + cfg = eval_only_script.load_config(eval_only_script.parse_args()) + assert cfg["optimizer_model"] == "deepseek-chat" + assert cfg["target_model"] == "deepseek-chat" + + # 3. Per-role compatible model overrides + with mock.patch("sys.argv", [ + "eval_only.py", + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + "--optimizer_openai_compatible_model", "opt-compat-model", + "--target_openai_compatible_model", "tgt-compat-model", + ]): + cfg = eval_only_script.load_config(eval_only_script.parse_args()) + assert cfg["optimizer_model"] == "opt-compat-model" + assert cfg["target_model"] == "tgt-compat-model" + + # 4. Explicit per-role overrides take precedence + with mock.patch("sys.argv", [ + "eval_only.py", + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + "--openai_compatible_model", "fallback-model", + "--optimizer_model", "explicit-opt", + "--target_model", "explicit-tgt", + ]): + cfg = eval_only_script.load_config(eval_only_script.parse_args()) + assert cfg["optimizer_model"] == "explicit-opt" + assert cfg["target_model"] == "explicit-tgt" + + +def test_eval_only_forwards_optimizer_qwen_chat_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: + raw = { + "model": {"backend": "qwen_chat"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f: + f.write("# Dummy Skill\n") + skill_path = f.name + + cli_args = [ + "--config", config_path, + "--skill", skill_path, + "--optimizer_qwen_chat_base_url", "http://opt-qwen:8000/v1", + "--optimizer_qwen_chat_api_key", "opt-qwen-key", + "--optimizer_qwen_chat_temperature", "0.2", + "--optimizer_qwen_chat_timeout_seconds", "90", + "--optimizer_qwen_chat_max_tokens", "2048", + "--optimizer_qwen_chat_enable_thinking", "true", + "--optimizer_qwen_chat_thinking_mode", "deep", + "--target_qwen_chat_base_url", "http://tgt-qwen:8000/v1", + ] + + qwen_kwargs: dict = {} + + def fake_configure_qwen(**kwargs): + qwen_kwargs.update(kwargs) + + monkeypatch.setattr(eval_only_script, "configure_qwen_chat", fake_configure_qwen) + + class StopExecution(Exception): + pass + + def stop_at_adapter(*args, **kwargs): + raise StopExecution() + + monkeypatch.setattr(eval_only_script, "get_adapter", stop_at_adapter) + + with mock.patch("sys.argv", ["eval_only.py"] + cli_args): + with pytest.raises(StopExecution): + eval_only_script.main() + + assert qwen_kwargs["optimizer_base_url"] == "http://opt-qwen:8000/v1" + assert qwen_kwargs["optimizer_api_key"] == "opt-qwen-key" + assert qwen_kwargs["optimizer_temperature"] == 0.2 + assert qwen_kwargs["optimizer_timeout_seconds"] == 90.0 + assert qwen_kwargs["optimizer_max_tokens"] == 2048 + assert qwen_kwargs["optimizer_enable_thinking"] is True + assert qwen_kwargs["optimizer_thinking_mode"] == "deep" + assert qwen_kwargs["target_base_url"] == "http://tgt-qwen:8000/v1" + + From d1ed2f2da738ea9b3800181c962454f521f3ec8d Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Mon, 31 Aug 2026 13:55:39 +0530 Subject: [PATCH 3/3] fix(model): initialize MiniMax OPTIMIZER_DEPLOYMENT at import and add fresh-import regression --- skillopt/model/__init__.py | 3 +- skillopt/model/minimax_backend.py | 46 ++++++++++- tests/test_minimax_backend.py | 130 ++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 4 deletions(-) diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py index 1d0edd02..28ade6fb 100644 --- a/skillopt/model/__init__.py +++ b/skillopt/model/__init__.py @@ -326,7 +326,7 @@ def chat_optimizer_messages( timeout=timeout, ) if get_optimizer_backend() == "minimax_chat": - return _minimax.chat_target_messages( + return _minimax.chat_optimizer_messages( messages=messages, max_completion_tokens=max_completion_tokens, retries=retries, @@ -821,5 +821,6 @@ def set_optimizer_deployment(deployment: str) -> None: _claude.set_optimizer_deployment(deployment) _claude_code.set_optimizer_deployment(deployment) _qwen.set_optimizer_deployment(deployment) + _minimax.set_optimizer_deployment(deployment) _openai_compat.set_optimizer_deployment(deployment) _codex.set_optimizer_deployment(deployment) diff --git a/skillopt/model/minimax_backend.py b/skillopt/model/minimax_backend.py index 9736a533..3737d292 100644 --- a/skillopt/model/minimax_backend.py +++ b/skillopt/model/minimax_backend.py @@ -69,6 +69,10 @@ def base_url_for_region(region: str | None) -> str: "TARGET_DEPLOYMENT", default_model_for_backend("minimax_chat"), ) +OPTIMIZER_DEPLOYMENT = os.environ.get( + "OPTIMIZER_DEPLOYMENT", + default_model_for_backend("minimax_chat"), +) # Models whose thinking cannot actually be turned off. Per MiniMax's # OpenAI-compatible docs the M2.x family accepts ``{"type": "disabled"}`` but @@ -304,7 +308,7 @@ def chat_target( stage: str = "target", reasoning_effort: str | None = None, timeout: float | None = None, -) -> tuple[str, dict[int]]: +) -> tuple[str, dict[str, int]]: del reasoning_effort messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] return _chat_messages_impl( @@ -312,6 +316,7 @@ def chat_target( max_completion_tokens, retries, stage, + deployment=TARGET_DEPLOYMENT, timeout=timeout, ) @@ -324,7 +329,7 @@ def chat_optimizer( stage: str = "optimizer", reasoning_effort: str | None = None, timeout: float | None = None, -) -> tuple[str, dict[int]]: +) -> tuple[str, dict[str, int]]: """Optimizer chat call. Backend stores the trained skill; uses the same MiniMax-proxied OpenAI-compat endpoint as `chat_target`. Added in the parallel-training fix; previously missing in skillopt 0.2.0's @@ -332,12 +337,14 @@ def chat_optimizer( (Azure) and produced "[skip] no usable patches" for any user running optimizer+target on `minimax_chat`. """ + del reasoning_effort messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] return _chat_messages_impl( messages, max_completion_tokens, retries, stage, + deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout, ) @@ -363,6 +370,33 @@ def chat_target_messages( tools=tools, tool_choice=tool_choice, return_message=return_message, + deployment=TARGET_DEPLOYMENT, + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + del reasoning_effort + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout, ) @@ -382,4 +416,10 @@ def set_reasoning_effort(effort: str | None) -> None: def set_target_deployment(deployment: str) -> None: global TARGET_DEPLOYMENT TARGET_DEPLOYMENT = deployment or default_model_for_backend("minimax_chat") - os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT \ No newline at end of file + os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT + + +def set_optimizer_deployment(deployment: str) -> None: + global OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment or default_model_for_backend("minimax_chat") + os.environ["OPTIMIZER_DEPLOYMENT"] = OPTIMIZER_DEPLOYMENT \ No newline at end of file diff --git a/tests/test_minimax_backend.py b/tests/test_minimax_backend.py index ec049a21..8c914c7b 100644 --- a/tests/test_minimax_backend.py +++ b/tests/test_minimax_backend.py @@ -2,8 +2,10 @@ from __future__ import annotations +import importlib import importlib.util import json +import os import sys import types from collections.abc import Iterator @@ -71,6 +73,7 @@ def minimax_backend() -> Iterator[Any]: snapshot = { "ENABLE_THINKING": backend.ENABLE_THINKING, "TARGET_DEPLOYMENT": backend.TARGET_DEPLOYMENT, + "OPTIMIZER_DEPLOYMENT": backend.OPTIMIZER_DEPLOYMENT, "API_KEY": backend.API_KEY, "BASE_URL": backend.BASE_URL, } @@ -91,6 +94,8 @@ def test_default_deployment_is_current_model(minimax_backend: Any) -> None: from skillopt.model.common import default_model_for_backend assert default_model_for_backend("minimax_chat") == "MiniMax-M3" + assert minimax_backend.TARGET_DEPLOYMENT == "MiniMax-M3" + assert minimax_backend.OPTIMIZER_DEPLOYMENT == "MiniMax-M3" def test_always_on_model_sends_adaptive_not_disabled( @@ -169,3 +174,128 @@ def test_unknown_deployment_defaults_to_adaptive( minimax_backend.chat_target("system", "user", retries=1) assert recorder.calls[0]["payload"]["thinking"] == {"type": "adaptive"} + + +def test_chat_optimizer_and_target_use_respective_deployments( + monkeypatch: pytest.MonkeyPatch, minimax_backend: Any +) -> None: + minimax_backend.TARGET_DEPLOYMENT = "MiniMax-Target-Model" + minimax_backend.OPTIMIZER_DEPLOYMENT = "MiniMax-Optimizer-Model" + recorder = _record_urlopen(monkeypatch, minimax_backend) + + minimax_backend.chat_target("sys_target", "user_target", retries=1) + minimax_backend.chat_optimizer("sys_opt", "user_opt", retries=1) + minimax_backend.chat_target_messages([{"role": "user", "content": "msg_target"}], retries=1) + minimax_backend.chat_optimizer_messages([{"role": "user", "content": "msg_opt"}], retries=1) + + assert recorder.calls[0]["payload"]["model"] == "MiniMax-Target-Model" + assert recorder.calls[1]["payload"]["model"] == "MiniMax-Optimizer-Model" + assert recorder.calls[2]["payload"]["model"] == "MiniMax-Target-Model" + assert recorder.calls[3]["payload"]["model"] == "MiniMax-Optimizer-Model" + + +def test_set_optimizer_and_target_deployment(minimax_backend: Any) -> None: + minimax_backend.set_target_deployment("MiniMax-New-Target") + assert minimax_backend.TARGET_DEPLOYMENT == "MiniMax-New-Target" + assert os.environ.get("TARGET_DEPLOYMENT") == "MiniMax-New-Target" + + minimax_backend.set_optimizer_deployment("MiniMax-New-Optimizer") + assert minimax_backend.OPTIMIZER_DEPLOYMENT == "MiniMax-New-Optimizer" + assert os.environ.get("OPTIMIZER_DEPLOYMENT") == "MiniMax-New-Optimizer" + + +def test_timeout_forwarded_to_urlopen( + monkeypatch: pytest.MonkeyPatch, minimax_backend: Any +) -> None: + recorder = _record_urlopen(monkeypatch, minimax_backend) + + minimax_backend.chat_target("system", "user", retries=1, timeout=42.5) + minimax_backend.chat_optimizer("system", "user", retries=1, timeout=55.0) + minimax_backend.chat_target_messages([{"role": "user", "content": "hi"}], retries=1, timeout=60.0) + minimax_backend.chat_optimizer_messages([{"role": "user", "content": "hi"}], retries=1, timeout=75.0) + + assert recorder.calls[0]["timeout"] == 42.5 + assert recorder.calls[1]["timeout"] == 55.0 + assert recorder.calls[2]["timeout"] == 60.0 + assert recorder.calls[3]["timeout"] == 75.0 + + +def test_fresh_import_optimizer_calls_without_setter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression: calling chat_optimizer or chat_optimizer_messages without calling + + set_optimizer_deployment() on a fresh import must not raise NameError for + OPTIMIZER_DEPLOYMENT. + """ + _install_openai_stub() + monkeypatch.delenv("OPTIMIZER_DEPLOYMENT", raising=False) + monkeypatch.delenv("TARGET_DEPLOYMENT", raising=False) + + from skillopt.model import minimax_backend as backend + + module = importlib.reload(backend) + recorder = _record_urlopen(monkeypatch, module) + + text, usage = module.chat_optimizer("system prompt", "user query", retries=1) + assert text == "answer" + assert recorder.calls[0]["payload"]["model"] == "MiniMax-M3" + + msg, usage_msg = module.chat_optimizer_messages( + [{"role": "user", "content": "user query"}], retries=1 + ) + assert msg == "answer" + assert recorder.calls[1]["payload"]["model"] == "MiniMax-M3" + + +def test_fresh_import_respects_optimizer_deployment_env( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_openai_stub() + monkeypatch.setenv("OPTIMIZER_DEPLOYMENT", "MiniMax-Env-Optimizer") + monkeypatch.setenv("TARGET_DEPLOYMENT", "MiniMax-Env-Target") + + from skillopt.model import minimax_backend as backend + + module = importlib.reload(backend) + recorder = _record_urlopen(monkeypatch, module) + + module.chat_optimizer("system prompt", "user query", retries=1) + module.chat_optimizer_messages( + [{"role": "user", "content": "user query"}], retries=1 + ) + module.chat_target("system prompt", "user query", retries=1) + module.chat_target_messages( + [{"role": "user", "content": "user query"}], retries=1 + ) + + assert recorder.calls[0]["payload"]["model"] == "MiniMax-Env-Optimizer" + assert recorder.calls[1]["payload"]["model"] == "MiniMax-Env-Optimizer" + assert recorder.calls[2]["payload"]["model"] == "MiniMax-Env-Target" + assert recorder.calls[3]["payload"]["model"] == "MiniMax-Env-Target" + + +def test_model_dispatcher_chat_optimizer_messages_minimax( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_openai_stub() + import skillopt.model as model + from skillopt.model import backend_config, minimax_backend as backend + + module = importlib.reload(backend) + recorder = _record_urlopen(monkeypatch, module) + + backend_config.set_optimizer_backend("minimax_chat") + model.set_optimizer_deployment("MiniMax-Custom-Opt") + + res, _ = model.chat_optimizer("system", "user", retries=1, timeout=99) + assert res == "answer" + assert recorder.calls[0]["payload"]["model"] == "MiniMax-Custom-Opt" + assert recorder.calls[0]["timeout"] == 99 + + res_msg, _ = model.chat_optimizer_messages( + [{"role": "user", "content": "test"}], retries=1, timeout=88 + ) + assert res_msg == "answer" + assert recorder.calls[1]["payload"]["model"] == "MiniMax-Custom-Opt" + assert recorder.calls[1]["timeout"] == 88