diff --git a/src/opendox/doxbench_binding.py b/src/opendox/doxbench_binding.py index fdd9e2ff..3b1e2131 100644 --- a/src/opendox/doxbench_binding.py +++ b/src/opendox/doxbench_binding.py @@ -93,21 +93,31 @@ AUTH_KINDS: tuple[str, ...] = (AUTH_KIND_API_KEY, AUTH_KIND_OAUTH) #: The CLOSED dialect vocabulary a binding may declare — the request grammar the -#: provider client speaks at the declared endpoint. ONE member today: this -#: repository's own already-declared turn shape, a prompt in and an -#: `assistant_prose` out, which is the shape `doxbench_model.dispatch_turn` -#: validates on the way back, so no second response grammar exists to keep -#: honest. CLOSED rather than open because an UNKNOWN dialect must REFUSE rather -#: than be guessed at: sending an assembled prompt to an endpoint whose grammar -#: this client does not know is a paid call that cannot succeed. A second member -#: joins here and an arm joins beside the first in `doxbench_provider`; the check -#: is never loosened. +#: provider client speaks at the declared endpoint. CLOSED rather than open +#: because an UNKNOWN dialect must REFUSE rather than be guessed at: sending an +#: assembled prompt to an endpoint whose grammar this client does not know is a +#: paid call that cannot succeed. A member joins here and an arm joins beside the +#: others in `doxbench_provider`; the check is never loosened. +#: +#: TWO MEMBERS, and the second joined exactly that way (#1144 box 16.1; plan 034 +#: T078): +#: +#: * `xfactory-prompt-v1` — this repository's own already-declared turn shape, +#: a POST of a model and a prompt answered by an `assistant_prose`, which is +#: the shape `doxbench_model.dispatch_turn` validates on the way back. It +#: stays FIRST, and it is unchanged byte for byte; +#: * `openai-chat-v1` — the OpenAI-compatible chat-completions grammar: a +#: request of a model and a list of messages, answered by the content of +#: the first choice's message. It is what a hosted API and the usual local +#: server both speak, which the first member does not. Its arm is in +#: `doxbench_provider` alone, beside the first one's. #: #: THE VOCABULARY LIVES HERE, on the record that declares it, and #: `doxbench_provider` reads it from this module — so an unknown dialect is #: refused when an operator DECLARES the binding rather than when a turn fails. DIALECT_XFACTORY_PROMPT_V1 = "xfactory-prompt-v1" -DIALECTS: tuple[str, ...] = (DIALECT_XFACTORY_PROMPT_V1,) +DIALECT_OPENAI_CHAT_V1 = "openai-chat-v1" +DIALECTS: tuple[str, ...] = (DIALECT_XFACTORY_PROMPT_V1, DIALECT_OPENAI_CHAT_V1) #: The URL schemes a declared endpoint may carry. `http://` is permitted for the #: on-this-host proxy posture an operator may legitimately run; a scheme this diff --git a/src/opendox/doxbench_provider.py b/src/opendox/doxbench_provider.py index fcff458b..42e9ad9e 100644 --- a/src/opendox/doxbench_provider.py +++ b/src/opendox/doxbench_provider.py @@ -34,7 +34,10 @@ console process. Brett's ruling of 2026-08-08: the broker mints, doxBench calls, because a broker in the request path adds a hop to every turn and to every chunk of a streamed one. WHERE to call and WHAT GRAMMAR to speak are - the BINDING's — the broker's declaration emits neither, deliberately; + the BINDING's — the broker's declaration emits neither, deliberately. Each + grammar the binding may declare has one arm here (`_DIALECT_ARMS`): this + repository's own prompt grammar, and the OpenAI-compatible chat-completions + grammar (#1144 box 16.1); * EXPIRY is handled by the 2026-08-26 ruling: re-mint and retry ONCE, with the re-mint and the paid retry visibly recorded, and a second expiry inside one turn surfaces the standard refusal rather than buying a third call. The @@ -175,8 +178,9 @@ #: it is the record that validates it). Aliased rather than respelled so the two #: modules cannot drift into two vocabularies. An unknown dialect is refused #: when an operator DECLARES the binding — earlier than a mint, and earlier than -#: a paid call. +#: a paid call. Each member has exactly one ARM below (`_DIALECT_ARMS`). DIALECT_XFACTORY_PROMPT_V1 = binding_mod.DIALECT_XFACTORY_PROMPT_V1 +DIALECT_OPENAI_CHAT_V1 = binding_mod.DIALECT_OPENAI_CHAT_V1 DIALECTS: tuple[str, ...] = binding_mod.DIALECTS #: How long a broker invocation may take. A mint is a local process doing local @@ -616,13 +620,27 @@ def list_references(binding, *, runner=subprocess_broker_runner) -> list: # the provider transport # --------------------------------------------------------------------------- -#: The provider request's own field names, in the ONE dialect this client -#: speaks. Named constants rather than inline literals so the boundary test can -#: assert they exist only here. +#: The provider request's own field names, per dialect. Named constants rather +#: than inline literals so the boundary test can assert they exist only here. +#: `model` is the one field both grammars share. PROVIDER_REQUEST_MODEL_FIELD = "model" + +#: `xfactory-prompt-v1`: a model and a prompt in, an `assistant_prose` out. PROVIDER_REQUEST_PROMPT_FIELD = "prompt" PROVIDER_RESPONSE_PROSE_FIELD = "assistant_prose" +#: `openai-chat-v1` (#1144 box 16.1; plan 034 T078): the chat-completions +#: request, a model and a list of messages, and its answer, the content of the +#: first choice's message. The assembled prompt travels as ONE message in the +#: user role. Prompt assembly is on the other side of the port (D14), so this +#: arm carries the text it was given and composes no message of its own. +PROVIDER_REQUEST_MESSAGES_FIELD = "messages" +CHAT_MESSAGE_ROLE_FIELD = "role" +CHAT_MESSAGE_CONTENT_FIELD = "content" +CHAT_ROLE_USER = "user" +CHAT_RESPONSE_CHOICES_FIELD = "choices" +CHAT_RESPONSE_MESSAGE_FIELD = "message" + #: The status a provider returns when the presented token is no longer good. #: 401 only: a 403 is an authorization verdict about what the token may do, #: which re-minting the same scope cannot change, and retrying it would buy a @@ -630,6 +648,62 @@ def list_references(binding, *, runner=subprocess_broker_runner) -> list: PROVIDER_STATUS_TOKEN_EXPIRED = 401 +def _prompt_request(model: str, prompt: str) -> dict: + """`xfactory-prompt-v1`'s request, exactly as it has always been sent.""" + return {PROVIDER_REQUEST_MODEL_FIELD: model, + PROVIDER_REQUEST_PROMPT_FIELD: prompt} + + +def _prompt_answer(document: dict) -> str: + """`xfactory-prompt-v1`'s answer: its `assistant_prose`, a string.""" + prose = document.get(PROVIDER_RESPONSE_PROSE_FIELD) + if not isinstance(prose, str): + raise BrokerRefused(DIAG_PROVIDER_MALFORMED) + return prose + + +def _chat_request(model: str, prompt: str) -> dict: + """`openai-chat-v1`'s request: the model, and the prompt as one message in + the user role.""" + return {PROVIDER_REQUEST_MODEL_FIELD: model, + PROVIDER_REQUEST_MESSAGES_FIELD: [ + {CHAT_MESSAGE_ROLE_FIELD: CHAT_ROLE_USER, + CHAT_MESSAGE_CONTENT_FIELD: prompt}]} + + +def _chat_answer(document: dict) -> str: + """`openai-chat-v1`'s answer: `choices[0].message.content`, a string. + + Read at exactly that path and nowhere else. A body with no first choice, a + choice with no message, or a message whose content is not text (a tool-call + answer carries null there) is not an answer this seam can hand back as + prose. Each lands on the fixed `DIAG_PROVIDER_MALFORMED` that every other + unusable answer lands on. Nothing past the first choice is read: the + request asks for one.""" + choices = document.get(CHAT_RESPONSE_CHOICES_FIELD) + if not isinstance(choices, list) or not choices: + raise BrokerRefused(DIAG_PROVIDER_MALFORMED) + first = choices[0] + message = (first.get(CHAT_RESPONSE_MESSAGE_FIELD) + if isinstance(first, dict) else None) + content = (message.get(CHAT_MESSAGE_CONTENT_FIELD) + if isinstance(message, dict) else None) + if not isinstance(content, str): + raise BrokerRefused(DIAG_PROVIDER_MALFORMED) + return content + + +#: ONE ARM PER DECLARED DIALECT: the function that builds its request and the +#: function that reads its answer. The record's closed vocabulary +#: (`doxbench_binding.DIALECTS`) refuses any other member at declaration, and a +#: test holds this table's keys equal to that vocabulary, so a member cannot +#: join one without the other. +_DIALECT_ARMS: dict[str, tuple] = { + DIALECT_XFACTORY_PROMPT_V1: (_prompt_request, _prompt_answer), + DIALECT_OPENAI_CHAT_V1: (_chat_request, _chat_answer), +} + + def _post_to_provider(token: MintedToken, *, model_id: str, prompt: str, timeout: float, opener) -> str: """The ONE place a provider is contacted. Returns the assistant prose. @@ -638,6 +712,11 @@ def _post_to_provider(token: MintedToken, *, model_id: str, prompt: str, it is not in the URL (which a proxy logs), not in the body (which an error handler might echo), and not in this function's return value. + THE GRAMMAR IS THE BINDING'S DIALECT (#1144 box 16.1), which the token + carries from the binding. Its arm in `_DIALECT_ARMS` builds the request + body and reads the answer. The route, the header, the bound, the expiry + status and every refusal below are the same for both dialects. + THE ANSWER IS BOUNDED (PR #392 review note b). `response.read()` with no argument reads until the peer stops sending, which makes the memory of this process a function of what a declared endpoint chooses to send — and the @@ -645,10 +724,14 @@ def _post_to_provider(token: MintedToken, *, model_id: str, prompt: str, byte over `MAX_PROVIDER_ANSWER_BYTES` is read deliberately, so an answer that is exactly at the bound is still honoured while one past it is detected rather than truncated into a shorter document that would parse.""" - body = json.dumps({ - PROVIDER_REQUEST_MODEL_FIELD: model_id, - PROVIDER_REQUEST_PROMPT_FIELD: prompt, - }).encode("utf-8") + arm = _DIALECT_ARMS.get(token.dialect) + if arm is None: + raise AssertionError( + f"{token.dialect!r} is outside the declared dialect vocabulary " + f"{DIALECTS}; the binding refuses it at declaration, so no turn " + "can carry one") + build_request, read_answer = arm + body = json.dumps(build_request(model_id, prompt)).encode("utf-8") request = urllib.request.Request( # noqa: S310 - endpoint declared on the binding by its operator, carried on the minted token token.endpoint, data=body, method="POST") request.add_header("Content-Type", "application/json") @@ -680,10 +763,7 @@ def _post_to_provider(token: MintedToken, *, model_id: str, prompt: str, raise BrokerRefused(DIAG_PROVIDER_MALFORMED) from error if not isinstance(document, dict): raise BrokerRefused(DIAG_PROVIDER_MALFORMED) - prose = document.get(PROVIDER_RESPONSE_PROSE_FIELD) - if not isinstance(prose, str): - raise BrokerRefused(DIAG_PROVIDER_MALFORMED) - return prose + return read_answer(document) # --------------------------------------------------------------------------- diff --git a/tests/test_model_provider_broker.py b/tests/test_model_provider_broker.py index 4a5fbaea..76cbd2ec 100644 --- a/tests/test_model_provider_broker.py +++ b/tests/test_model_provider_broker.py @@ -21,6 +21,9 @@ redacted refusal `doxbench_model.dispatch_turn` already defines, and the UNCONFIGURED posture is byte-for-byte what it was before this change. +A SIXTH LAYER, (f), holds #1144 Group 16's binding and provider boxes (plan +034 phase 3, slice P3-B). 16.1 is the OpenAI-compatible dialect (T078). + THE FAKE BROKER SPEAKS THE DECLARED CONTRACT (task 2.6). It was this repository's own invented stdin/stdout protocol until the reconciliation, which meant every test here agreed with a broker that does not exist. It now takes the @@ -40,6 +43,7 @@ from __future__ import annotations +import contextlib import dataclasses import http.server import io @@ -145,8 +149,12 @@ def test_the_dialect_vocabulary_is_closed_and_refuses_at_declaration(): there; openProfiler's declaration emits no dialect at all, so the fact is the BINDING's and the refusal happens when an operator DECLARES one — before any broker is invoked and long before a paid call. Closed, still: an unknown - grammar refuses rather than being guessed at.""" - assert binding_mod.DIALECTS == ("xfactory-prompt-v1",) + grammar refuses rather than being guessed at. + + TWO MEMBERS since #1144 box 16.1 (plan 034 T078), and the order is pinned: + the prompt grammar stays first, and the OpenAI-compatible chat grammar + joins after it. The refusal below is the same refusal it always was.""" + assert binding_mod.DIALECTS == ("xfactory-prompt-v1", "openai-chat-v1") assert provider_mod.DIALECTS is binding_mod.DIALECTS, \ "one vocabulary, read from the record that declares it" with pytest.raises(binding_mod.BindingRefused) as caught: @@ -866,9 +874,9 @@ def _expired_error(): def _port(tmp_path, *outcomes, expires=None, notice=None, clock=time.time, - endpoint=ENDPOINT): + endpoint=ENDPOINT, dialect=binding_mod.DIALECT_XFACTORY_PROMPT_V1): script = _write_broker(tmp_path, expires=expires) - binding = _broker_binding(script, endpoint=endpoint) + binding = _broker_binding(script, endpoint=endpoint, dialect=dialect) opener = _Opener(*outcomes) port = provider_mod.BrokeredProviderPort( binding, install_mod.brokered_catalog(binding), @@ -1336,3 +1344,241 @@ def test_the_subprocess_runner_never_uses_a_shell(tmp_path): assert "shell=True" not in source assert "os.system" not in source assert subprocess.Popen is subprocess.Popen # the module spawns, nothing else + + +# =========================================================================== +# (f) CHAT'S MODEL CONFIGURATION (#1144 Group 16; plan 034 phase 3, P3-B) +# =========================================================================== +# +# 16.1, the OpenAI-compatible dialect (T078). `openai-chat-v1` is the second +# `DIALECTS` member. Its request is the chat-completions grammar (`model`, +# `messages`), and its answer is read at `choices[0].message.content`. Both are +# spoken by one arm in `doxbench_provider`, beside the prompt grammar's arm. + +OPENAI_CHAT = binding_mod.DIALECT_OPENAI_CHAT_V1 + + +def _chat_completion(content="the chat answer"): + """A chat-completions answer in that grammar's own shape. The keys around + `choices` are what a real server sends, and nothing here reads them.""" + return {"id": "chatcmpl-stand-in", "object": "chat.completion", + "model": "stand-in-model", + "choices": [{"index": 0, "finish_reason": "stop", + "message": {"role": "assistant", + "content": content}}]} + + +def test_f16_1_the_openai_compatible_dialect_is_declared(): + """F16.1's dialect assertion, as #1144 writes it: + `assert "openai-chat-v1" in b.DIALECTS`. A binding may declare it.""" + assert "openai-chat-v1" in binding_mod.DIALECTS, ( + f"no OpenAI-compatible dialect: {binding_mod.DIALECTS}") + assert OPENAI_CHAT == "openai-chat-v1" + assert _binding(dialect=OPENAI_CHAT).dialect == OPENAI_CHAT + assert provider_mod.DIALECT_OPENAI_CHAT_V1 is OPENAI_CHAT, \ + "one spelling, read from the record that declares it" + + +def test_every_declared_dialect_has_exactly_one_arm_in_the_provider_module(): + """A member cannot join the vocabulary without an arm, or an arm exist + for a member the record would refuse.""" + assert set(provider_mod._DIALECT_ARMS) == set(binding_mod.DIALECTS) + + +def test_a_chat_turn_speaks_the_chat_completions_grammar(tmp_path): + port, opener = _port(tmp_path, _chat_completion("the answer"), + dialect=OPENAI_CHAT) + assert port.dispatch(_Envelope()) == {"assistant_prose": "the answer", + "proposals": []} + request = opener.requests[0] + assert request.get_method() == "POST" + assert request.get_full_url() == ENDPOINT + assert json.loads(request.data.decode("utf-8")) == { + "model": "openprofiler-demo", + "messages": [{"role": "user", "content": "assembled prompt"}]} + assert request.get_header("Content-type") == "application/json" + # the token travels in the header, exactly as it does for the prompt grammar + assert request.get_header("Authorization") == f"Bearer {SENTINEL_TOKEN}" + assert SENTINEL_TOKEN not in request.get_full_url() + assert SENTINEL_TOKEN not in request.data.decode("utf-8") + + +def test_the_prompt_dialect_is_unchanged_byte_for_byte(tmp_path): + """The first member's request is the bytes it always was: the arm table + moved the code, and nothing it sends.""" + port, opener = _port(tmp_path, {"assistant_prose": "a"}) + assert port.dispatch(_Envelope())["assistant_prose"] == "a" + assert opener.requests[0].data == json.dumps( + {"model": "openprofiler-demo", "prompt": "assembled prompt"} + ).encode("utf-8") + + +@pytest.mark.parametrize("answer", [ + {}, + {"choices": []}, + {"choices": "not a list"}, + {"choices": ["not an object"]}, + {"choices": [{}]}, + {"choices": [{"message": "not an object"}]}, + {"choices": [{"message": {"role": "assistant"}}]}, + {"choices": [{"message": {"role": "assistant", "content": None}}]}, + {"choices": [{"message": {"role": "assistant", "content": 7}}]}, + {"assistant_prose": "the prompt grammar's answer, not this one's"}, +], ids=["empty", "no-choice", "choices-not-a-list", "choice-not-an-object", + "no-message", "message-not-an-object", "no-content", "null-content", + "content-not-text", "the-other-grammar"]) +def test_a_chat_answer_off_the_declared_path_is_malformed(tmp_path, answer): + port, _opener = _port(tmp_path, answer, dialect=OPENAI_CHAT) + envelope = _Envelope() + with pytest.raises(provider_mod.BrokerRefused) as caught: + port.dispatch(envelope) + assert caught.value.diagnostic == provider_mod.DIAG_PROVIDER_MALFORMED + + +def test_a_chat_shaped_answer_is_not_the_prompt_grammars_answer(tmp_path): + """Each arm reads its own grammar and no other.""" + port, _opener = _port(tmp_path, _chat_completion()) + envelope = _Envelope() + with pytest.raises(provider_mod.BrokerRefused) as caught: + port.dispatch(envelope) + assert caught.value.diagnostic == provider_mod.DIAG_PROVIDER_MALFORMED + + +def test_only_the_first_choice_is_read(tmp_path): + answer = _chat_completion("first") + answer["choices"].append({"index": 1, "finish_reason": "stop", + "message": {"role": "assistant", + "content": "second"}}) + port, _opener = _port(tmp_path, answer, dialect=OPENAI_CHAT) + assert port.dispatch(_Envelope())["assistant_prose"] == "first" + + +def test_the_expiry_ruling_holds_for_the_chat_grammar(tmp_path): + """The 2026-08-26 ruling is the port's, not a dialect's: a mid-turn expiry + re-mints and retries once, visibly, in either grammar.""" + printed: list[str] = [] + port, opener = _port(tmp_path, _expired_error(), + _chat_completion("the retried answer"), + notice=printed.append, dialect=OPENAI_CHAT) + assert port.dispatch(_Envelope())["assistant_prose"] == "the retried answer" + assert len(opener.requests) == 2, "exactly one paid retry" + assert [event.reason for event in port.ledger] == [ + provider_mod.REASON_FIRST_MINT, + provider_mod.REASON_EXPIRY_REMINT, + provider_mod.REASON_PAID_RETRY, + ] + assert printed + assert "re-minted once and retried" in printed[0] + + +def test_the_answer_bound_holds_for_the_chat_grammar(tmp_path): + bound = provider_mod.MAX_PROVIDER_ANSWER_BYTES + oversize = json.dumps(_chat_completion("x" * bound)).encode("utf-8") + port, _opener = _port(tmp_path, oversize, dialect=OPENAI_CHAT) + envelope = _Envelope() + with pytest.raises(provider_mod.BrokerRefused) as caught: + port.dispatch(envelope) + assert caught.value.diagnostic == provider_mod.DIAG_PROVIDER_MALFORMED + + +def test_a_chat_provider_refusal_lands_on_the_fixed_sentence(tmp_path): + port, _opener = _port( + tmp_path, + urllib.error.HTTPError(ENDPOINT, 400, "Bad Request", {}, + io.BytesIO(b'{"error":{"message":"leaky"}}')), + dialect=OPENAI_CHAT) + envelope = _Envelope() + with pytest.raises(provider_mod.BrokerRefused) as caught: + port.dispatch(envelope) + assert caught.value.diagnostic == provider_mod.DIAG_PROVIDER_REFUSED + assert "leaky" not in str(caught.value) + + +@contextlib.contextmanager +def _stand_in_provider(handler_class): + """A stand-in provider on loopback for the length of one test. It yields + the server's base URL, and it is shut down and joined however the test + ends.""" + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler_class) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, prt = server.server_address[:2] + yield f"http://{host}:{prt}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _answer_json(handler, document) -> None: + """Answer one stand-in request with `document` as a JSON body.""" + payload = json.dumps(document).encode("utf-8") + handler.send_response(200) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(payload))) + handler.end_headers() + handler.wfile.write(payload) + + +class _ChatCompletionsHandler(http.server.BaseHTTPRequestHandler): + """A stand-in OpenAI-compatible server on loopback. It records each + request and answers in the chat-completions grammar.""" + + seen: dict = {} + + def do_POST(self): # noqa: N802 - BaseHTTPRequestHandler's own spelling + length = int(self.headers.get("Content-Length", "0")) + _ChatCompletionsHandler.seen = { + "path": self.path, + "authorization": self.headers.get("Authorization"), + "content_type": self.headers.get("Content-Type"), + "body": json.loads(self.rfile.read(length).decode("utf-8")), + } + _answer_json(self, _chat_completion("answered in the chat grammar")) + + def log_message(self, *_args): + return + + +def test_a_chat_turn_reaches_a_stand_in_chat_completions_server(tmp_path): + """The real `urllib` path, in the chat grammar: a stand-in server on + loopback receives the request at the binding's declared endpoint, in that + grammar, with the token in the authorization header and nowhere else.""" + with _stand_in_provider(_ChatCompletionsHandler) as base: + binding = _broker_binding(_write_broker(tmp_path), + endpoint=f"{base}/v1/chat/completions", + dialect=OPENAI_CHAT) + port = provider_mod.BrokeredProviderPort( + binding, install_mod.brokered_catalog(binding), + notice=lambda _text: None) + assert port.dispatch(_Envelope()) == { + "assistant_prose": "answered in the chat grammar", + "proposals": []} + + seen = _ChatCompletionsHandler.seen + assert seen["path"] == "/v1/chat/completions" + assert seen["authorization"] == f"Bearer {SENTINEL_TOKEN}" + assert seen["content_type"] == "application/json" + assert seen["body"] == { + "model": "openprofiler-demo", + "messages": [{"role": "user", "content": "assembled prompt"}]} + assert SENTINEL_TOKEN not in json.dumps(seen["body"]) + + +def test_the_cli_declares_a_chat_binding(tmp_path, capsys): + """The operator door offers the dialect, because its choices are read from + the record's vocabulary rather than respelled.""" + checkout = tmp_path / "checkout" + checkout.mkdir() + args = cli_mod.build_parser().parse_args([ + "model-binding", "add", "--repo-root", str(checkout), + "--id", "local-chat", "--label", "Local chat", "--provider", "local", + "--credential-ref", FAKE_REFERENCE, "--auth-kind", "api_key", + "--credential-approver", "brett@opensoft.one", + "--endpoint", "http://127.0.0.1:9/v1/chat/completions", + "--dialect", OPENAI_CHAT, "--", "openprofiler-broker"]) + assert args.func(args) == 0 + capsys.readouterr() + store = binding_mod.BindingStore(binding_mod.bindings_path(checkout)) + assert store.get("local-chat").dialect == OPENAI_CHAT