From 9b3df36d4a5df66fdba236df2bc9dc20a58b3db5 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 23 Aug 2026 23:22:27 +0000 Subject: [PATCH 1/5] feat(oar): support templated document and repository reviews --- .../src/openshell_agent_runner/cli.py | 47 +++- .../src/openshell_agent_runner/config.py | 34 ++- .../harnesses/pi/resources.py | 12 +- .../profiles/reviewer/profile.yaml | 25 ++- .../profiles/reviewer/prompt-document.md | 11 + .../profiles/reviewer/prompt-repository.md | 11 + .../profiles/reviewer/prompt.md | 5 - .../prompt_templates.py | 48 +++++ .../src/openshell_agent_runner/runner.py | 140 ++++++++++-- .../tests/harnesses/test_pi.py | 53 ++++- .../openshell-agent-runner/tests/test_cli.py | 91 +++++++- .../tests/test_config.py | 35 ++- .../tests/test_profile_init.py | 3 + .../tests/test_prompt_templates.py | 38 ++++ .../tests/test_resolution.py | 201 ++++++++++++++++++ 15 files changed, 700 insertions(+), 54 deletions(-) create mode 100644 projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt-document.md create mode 100644 projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt-repository.md delete mode 100644 projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt.md create mode 100644 projects/openshell-agent-runner/src/openshell_agent_runner/prompt_templates.py create mode 100644 projects/openshell-agent-runner/tests/test_prompt_templates.py diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/cli.py b/projects/openshell-agent-runner/src/openshell_agent_runner/cli.py index 4e4950f..f19bb2e 100644 --- a/projects/openshell-agent-runner/src/openshell_agent_runner/cli.py +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/cli.py @@ -120,9 +120,16 @@ def run( output: Annotated[ Path, typer.Option("--output", help="Host path for the agent result.") ], - input_document: Annotated[ + input_path: Annotated[ Path | None, - typer.Option("--input", help="Host document required by document tasks."), + typer.Option("--input", help="Host input required by the selected task."), + ] = None, + prompt_variable: Annotated[ + list[str] | None, + typer.Option( + "--prompt-var", + help="Non-secret NAME=VALUE prompt variable. Repeat for several.", + ), ] = None, upload: Annotated[ list[str] | None, @@ -159,7 +166,8 @@ def run( profile_directory=profile, task_id=task, output=output, - input_document=input_document, + input_path=input_path, + prompt_variables=prompt_variable or (), uploads=upload or (), environments=environment or (), gateway=gateway, @@ -227,8 +235,10 @@ def _render_task_help( _help_command(f" oar run {shlex.quote(str(profile_directory))} \\"), _help_command(f" --task {shlex.quote(task_id)} \\"), ] - if task.required_input == "document": - usage_lines.append(_help_command(" --input DOCUMENT \\")) + if task.required_input is not None: + usage_lines.append( + _help_command(f" --input {task.required_input.upper()} \\") + ) usage_lines.append(_help_command(" --output OUTPUT")) upload_lines = [_help_heading("Additional configured uploads:")] @@ -244,6 +254,22 @@ def _render_task_help( environment_lines.append(" None. Add values with --env KEY=VALUE.") input_lines = _required_input_help(task.required_input) + prompt_variable_lines = [_help_heading("Prompt variables:")] + if task.prompt_variables: + for name, variable in task.prompt_variables.items(): + requirement = ( + f"Default: {variable.default}" + if variable.default is not None + else "Required." + ) + prompt_variable_lines.extend( + [ + _help_command(f" --prompt-var {name}=VALUE"), + f" {variable.description} {requirement}", + ] + ) + else: + prompt_variable_lines.append(" None.") output_description = ( f"JSON validated against {task.output_schema}." @@ -260,6 +286,8 @@ def _render_task_help( "", *input_lines, "", + *prompt_variable_lines, + "", *upload_lines, "", *environment_lines, @@ -274,10 +302,15 @@ def _render_task_help( def _required_input_help(required_input: str | None) -> list[str]: if required_input is None: return [_help_heading("Required input:"), " None."] + description = ( + "Host document to review." + if required_input == "document" + else "Host code repository to review." + ) return [ _help_heading("Required argument:"), - _help_command(" --input DOCUMENT"), - " Host document to review.", + _help_command(f" --input {required_input.upper()}"), + f" {description}", ] diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/config.py b/projects/openshell-agent-runner/src/openshell_agent_runner/config.py index a66ff44..f85af65 100644 --- a/projects/openshell-agent-runner/src/openshell_agent_runner/config.py +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/config.py @@ -23,6 +23,11 @@ ) from openshell_agent_runner.errors import ConfigurationError +from openshell_agent_runner.prompt_templates import ( + BUILTIN_PROMPT_VARIABLES, + PROMPT_VARIABLE_NAME_PATTERN, + validate_prompt_template, +) IDENTIFIER_PATTERN = r"^[a-z][a-z0-9-]{0,62}$" RESOURCE_IDENTIFIER_PATTERN = r"^[a-z][a-z0-9_-]{0,62}$" @@ -62,6 +67,7 @@ def validate_environment(cls, values: list[str]) -> list[str]: ToolName = Annotated[str, Field(pattern=RESOURCE_IDENTIFIER_PATTERN)] +PromptVariableName = Annotated[str, Field(pattern=PROMPT_VARIABLE_NAME_PATTERN)] class ExtensionConfig(StrictModel): @@ -76,10 +82,18 @@ def require_unique_tools(cls, values: list[str]) -> list[str]: return values +class PromptVariableConfig(StrictModel): + description: str = Field(min_length=1, max_length=1000) + default: str | None = Field(default=None, min_length=1) + + class TaskConfig(StrictModel): description: str | None = Field(default=None, min_length=1, max_length=1000) - required_input: Literal["document"] | None = None + required_input: Literal["document", "repository"] | None = None prompt: Path + prompt_variables: dict[PromptVariableName, PromptVariableConfig] = Field( + default_factory=dict + ) output_schema: Path | None = None tools: list[ToolName] = Field(default_factory=list) skills: list[Path] = Field(default_factory=list) @@ -345,7 +359,23 @@ def _validate_profile_resources(resolved: ResolvedProfile) -> None: directory = resolved.profile_dir _inside(directory, directory / resolved.profile.sandbox.policy, "sandbox policy") for task_id, task in resolved.profile.tasks.items(): - _inside(directory, directory / task.prompt, f"prompt for task {task_id}") + prompt = _inside( + directory, directory / task.prompt, f"prompt for task {task_id}" + ) + available_builtins = ( + BUILTIN_PROMPT_VARIABLES if task.required_input is not None else frozenset() + ) + try: + template = prompt.read_text(encoding="utf-8") + validate_prompt_template( + template, + task.prompt_variables.keys(), + available_builtins, + ) + except (OSError, UnicodeError, ValueError) as error: + raise ConfigurationError( + f"invalid prompt template for task {task_id}: {error}" + ) from error if task.output_schema is not None: schema = _inside( directory, diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/harnesses/pi/resources.py b/projects/openshell-agent-runner/src/openshell_agent_runner/harnesses/pi/resources.py index 09dd40c..17631ed 100644 --- a/projects/openshell-agent-runner/src/openshell_agent_runner/harnesses/pi/resources.py +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/harnesses/pi/resources.py @@ -6,6 +6,7 @@ import json import shutil import tempfile +from collections.abc import Mapping from importlib.resources import files from pathlib import Path @@ -15,6 +16,7 @@ ResolvedProfile, ) from openshell_agent_runner.harnesses.resources import PreparedResources +from openshell_agent_runner.prompt_templates import render_prompt_template SANDBOX_RUNTIME_ROOT = "/sandbox/oar-runtime" @@ -23,13 +25,19 @@ def image_directory() -> Path: return Path(str(files("openshell_agent_runner.harnesses.pi") / "runtime" / "image")) -def prepare_resources(resolved: ResolvedProfile, task_id: str) -> PreparedResources: +def prepare_resources( + resolved: ResolvedProfile, + task_id: str, + prompt_variables: Mapping[str, str] | None = None, +) -> PreparedResources: temporary = tempfile.TemporaryDirectory(prefix="oar-pi-") runtime = Path(temporary.name) / "runtime" (runtime / "skills").mkdir(parents=True, exist_ok=True) (runtime / "extensions").mkdir(parents=True, exist_ok=True) task = resolved.profile.tasks[task_id] - shutil.copy2(resolved.profile_dir / task.prompt, runtime / "prompt.md") + template = (resolved.profile_dir / task.prompt).read_text(encoding="utf-8") + rendered_prompt = render_prompt_template(template, prompt_variables or {}) + (runtime / "prompt.md").write_text(rendered_prompt, encoding="utf-8") shutil.copy2(resolved.profile_dir / MODELS_FILENAME, runtime / MODELS_FILENAME) shutil.copy2(resolved.profile_dir / SETTINGS_FILENAME, runtime / SETTINGS_FILENAME) arguments = [ diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/profile.yaml b/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/profile.yaml index 89f72cf..431d19b 100644 --- a/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/profile.yaml +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/profile.yaml @@ -1,12 +1,31 @@ id: reviewer -description: Review a required input document and publish the result. +description: Review a required input document or code repository and publish the result. sandbox: policy: policy.yaml tasks: - review: + review-document: description: Review an input document and return a useful written result. required_input: document - prompt: prompt.md + prompt: prompt-document.md + prompt_variables: + focus: + description: Areas of the document that deserve special attention. + default: Review the complete document. + context: + description: Additional context that should inform the review. + default: No additional context was provided. + tools: [read, grep, find, ls, bash] + review-repository: + description: Review an input code repository and return a useful written result. + required_input: repository + prompt: prompt-repository.md + prompt_variables: + focus: + description: Files or directories that deserve special attention. + default: Review the entire repository. + context: + description: Additional context that should inform the review. + default: No additional context was provided. tools: [read, grep, find, ls, bash] diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt-document.md b/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt-document.md new file mode 100644 index 0000000..26787d3 --- /dev/null +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt-document.md @@ -0,0 +1,11 @@ +# Review the input document + +Act as a coding agent. Inspect `{{ oar.input_path }}`, originally provided as +`{{ oar.input_name }}`, using the declared tools as needed. + +Review focus: {{ focus }} + +Additional context: {{ context }} + +Return a concise Markdown review that identifies the document's strengths and +the most useful improvements to its clarity and completeness. diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt-repository.md b/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt-repository.md new file mode 100644 index 0000000..5d52e55 --- /dev/null +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt-repository.md @@ -0,0 +1,11 @@ +# Review the input code repository + +Inspect the code repository at `{{ oar.input_path }}`, originally provided as +`{{ oar.input_name }}`, using the declared tools as needed. + +Review focus: {{ focus }} + +Additional context: {{ context }} + +Return a concise Markdown review of material issues, including relevant file +paths and line references. Do not edit the repository. diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt.md b/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt.md deleted file mode 100644 index 02ef2e1..0000000 --- a/projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer/prompt.md +++ /dev/null @@ -1,5 +0,0 @@ -# Review the input document - -Act as a coding agent. Inspect `/workspace/input/document.md`, using the declared -tools as needed. Return a concise Markdown review that identifies the document's -strengths and the most useful improvements to its clarity and completeness. diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/prompt_templates.py b/projects/openshell-agent-runner/src/openshell_agent_runner/prompt_templates.py new file mode 100644 index 0000000..c215580 --- /dev/null +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/prompt_templates.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate and render the deliberately small OAR prompt template syntax.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Set + +BUILTIN_PROMPT_VARIABLES = frozenset( + { + "oar.input_name", + "oar.input_path", + } +) +PROMPT_VARIABLE_NAME_PATTERN = r"^[a-z][a-z0-9_]{0,62}$" +_PLACEHOLDER_PATTERN = re.compile(r"{{\s*((?:oar\.)?[a-z][a-z0-9_]{0,62})\s*}}") + + +def validate_prompt_template( + template: str, + declared_variables: Set[str], + available_builtins: Set[str], +) -> None: + placeholders = _prompt_placeholders(template) + unknown = sorted(placeholders - declared_variables - available_builtins) + if unknown: + raise ValueError(f"unknown prompt template variables: {unknown}") + unused = sorted(declared_variables - placeholders) + if unused: + raise ValueError(f"unused prompt variable declarations: {unused}") + + +def render_prompt_template(template: str, values: Mapping[str, str]) -> str: + placeholders = _prompt_placeholders(template) + missing = sorted(placeholders - values.keys()) + if missing: + raise ValueError(f"missing prompt template variables: {missing}") + return _PLACEHOLDER_PATTERN.sub(lambda match: values[match.group(1)], template) + + +def _prompt_placeholders(template: str) -> set[str]: + placeholders = {match.group(1) for match in _PLACEHOLDER_PATTERN.finditer(template)} + unmatched = _PLACEHOLDER_PATTERN.sub("", template) + if "{{" in unmatched or "}}" in unmatched: + raise ValueError("malformed prompt template placeholder") + return placeholders diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/runner.py b/projects/openshell-agent-runner/src/openshell_agent_runner/runner.py index 8dc46cd..7855ff4 100644 --- a/projects/openshell-agent-runner/src/openshell_agent_runner/runner.py +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/runner.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import re import secrets import shlex import sys @@ -22,6 +23,7 @@ ) from openshell_agent_runner.config import ( ResolvedProfile, + TaskConfig, resolve_task, validate_environment_assignments, validate_upload_mappings, @@ -31,6 +33,7 @@ image_directory, prepare_resources, ) +from openshell_agent_runner.prompt_templates import PROMPT_VARIABLE_NAME_PATTERN @dataclass(frozen=True) @@ -38,7 +41,8 @@ class RunRequest: profile_directory: Path task_id: str output: Path - input_document: Path | None = None + input_path: Path | None = None + prompt_variables: Sequence[str] = () uploads: Sequence[str] = () environments: Sequence[str] = () gateway: str | None = None @@ -48,29 +52,41 @@ class RunRequest: openshell_bin: str = "openshell" +@dataclass(frozen=True) +class ResolvedInput: + source: Path + sandbox_path: str + name: str + + @dataclass(frozen=True) class ResolvedRun: request: RunRequest profile: ResolvedProfile + input: ResolvedInput | None uploads: tuple[str, ...] environments: tuple[str, ...] + prompt_variables: tuple[tuple[str, str], ...] create_command: tuple[str, ...] def resolve_run(request: RunRequest) -> ResolvedRun: profile = resolve_task(request.profile_directory, request.task_id) task = profile.profile.tasks[request.task_id] - document_upload = _resolve_document_upload(request, task.required_input) + resolved_input, input_upload, input_environment = _resolve_required_input( + request, task.required_input + ) + prompt_variables = _resolve_prompt_variables(request, task, resolved_input) uploads = _validate_uploads( [ *profile.profile.sandbox.upload, - *([document_upload] if document_upload else []), + *([input_upload] if input_upload else []), *request.uploads, ] ) environments = _validate_environments( [ - *([_DOCUMENT_INPUT_ENVIRONMENT] if document_upload else []), + *([input_environment] if input_environment else []), *profile.profile.sandbox.env, *request.environments, ] @@ -95,8 +111,10 @@ def resolve_run(request: RunRequest) -> ResolvedRun: return ResolvedRun( request=request, profile=profile, + input=resolved_input, uploads=uploads, environments=environments, + prompt_variables=prompt_variables, create_command=tuple(command), ) @@ -105,7 +123,11 @@ def render_dry_run(request: RunRequest) -> str: """Render the exact nominal command sequence without executing subprocesses.""" resolved = resolve_run(request) name, token = _identity() - resources = prepare_resources(resolved.profile, request.task_id) + resources = prepare_resources( + resolved.profile, + request.task_id, + dict(resolved.prompt_variables), + ) try: with tempfile.TemporaryDirectory(prefix="oar-output-") as directory: downloaded = Path(directory) / "output.download" @@ -167,7 +189,11 @@ def render_dry_run(request: RunRequest) -> str: def run_agent(request: RunRequest) -> str: resolved = resolve_run(request) name, token = _identity() - resources = prepare_resources(resolved.profile, request.task_id) + resources = prepare_resources( + resolved.profile, + request.task_id, + dict(resolved.prompt_variables), + ) create = openshell.sandbox_create(resolved, name, token) primary_error: BaseException | None = None try: @@ -229,26 +255,104 @@ def _validate_environments(values: Sequence[str]) -> tuple[str, ...]: raise ConfigurationError(str(error)) from error -def _resolve_document_upload( +def _resolve_required_input( request: RunRequest, required_input: str | None -) -> str | None: +) -> tuple[ResolvedInput | None, str | None, str | None]: if required_input is None: - if request.input_document is not None: + if request.input_path is not None: raise ConfigurationError( f"task {request.task_id!r} does not accept --input" ) - return None - if request.input_document is None: - raise ConfigurationError(f"task {request.task_id!r} requires --input DOCUMENT") + return None, None, None + input_label = required_input.upper() + if request.input_path is None: + raise ConfigurationError( + f"task {request.task_id!r} requires --input {input_label}" + ) try: - document = request.input_document.resolve(strict=True) + input_path = request.input_path.resolve(strict=True) except OSError as error: raise ConfigurationError( - f"input document does not exist: {request.input_document}" + f"input {required_input} does not exist: {request.input_path}" ) from error - if not document.is_file(): - raise ConfigurationError(f"input document must be a file: {document}") - return f"{document}:{_DOCUMENT_INPUT_PATH}" + if required_input == "document": + if not input_path.is_file(): + raise ConfigurationError(f"input document must be a file: {input_path}") + suffix = input_path.suffix + if not ( + 1 < len(suffix) <= 17 and suffix.startswith(".") and suffix[1:].isalnum() + ): + suffix = "" + sandbox_input = f"{_INPUT_DIRECTORY}/document{suffix}" + resolved_input = ResolvedInput( + source=input_path, + sandbox_path=sandbox_input, + name=input_path.name, + ) + return ( + resolved_input, + f"{input_path}:{sandbox_input}", + _DOCUMENT_INPUT_ENVIRONMENT, + ) + if not input_path.is_dir(): + raise ConfigurationError(f"input repository must be a directory: {input_path}") + repository_root = f"{_INPUT_DIRECTORY}/{input_path.name}" + resolved_input = ResolvedInput( + source=input_path, + sandbox_path=repository_root, + name=input_path.name, + ) + return ( + resolved_input, + f"{input_path}:{_INPUT_DIRECTORY}", + f"REPOSITORY_ROOT={repository_root}", + ) + + +def _resolve_prompt_variables( + request: RunRequest, + task: TaskConfig, + resolved_input: ResolvedInput | None, +) -> tuple[tuple[str, str], ...]: + supplied: dict[str, str] = {} + for assignment in request.prompt_variables: + name, separator, value = assignment.partition("=") + if not separator or not value: + raise ConfigurationError( + "prompt variables must use non-empty NAME=VALUE syntax" + ) + if name.startswith("oar."): + raise ConfigurationError( + f"prompt variable uses reserved oar namespace: {name!r}" + ) + if not re.fullmatch(PROMPT_VARIABLE_NAME_PATTERN, name): + raise ConfigurationError(f"invalid prompt variable name: {name!r}") + if name in supplied: + raise ConfigurationError(f"duplicate prompt variable: {name!r}") + if name not in task.prompt_variables: + raise ConfigurationError(f"undeclared prompt variable: {name!r}") + supplied[name] = value + + values: dict[str, str] = {} + missing: list[str] = [] + for name, config in task.prompt_variables.items(): + value = supplied.get(name, config.default) + if value is None: + missing.append(name) + else: + values[name] = value + if missing: + raise ConfigurationError( + f"missing required prompt variables: {sorted(missing)}" + ) + if resolved_input is not None: + values.update( + { + "oar.input_name": resolved_input.name, + "oar.input_path": resolved_input.sandbox_path, + } + ) + return tuple(values.items()) def _validation_preview(resolved: ResolvedRun, downloaded: Path) -> str: @@ -286,5 +390,5 @@ def _verify_ownership(request: RunRequest, name: str, token: str) -> None: ) -_DOCUMENT_INPUT_PATH = "/workspace/input/document.md" _DOCUMENT_INPUT_ENVIRONMENT = "REPOSITORY_ROOT=/workspace/input" +_INPUT_DIRECTORY = "/workspace/input" diff --git a/projects/openshell-agent-runner/tests/harnesses/test_pi.py b/projects/openshell-agent-runner/tests/harnesses/test_pi.py index 7477302..b4b9d33 100644 --- a/projects/openshell-agent-runner/tests/harnesses/test_pi.py +++ b/projects/openshell-agent-runner/tests/harnesses/test_pi.py @@ -122,15 +122,39 @@ def test_plain_task_uses_final_response_without_submission_tool() -> None: REPOSITORY / "projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer" ) - prepared = prepare_resources(resolved, "review") - try: - assert "submit_result" not in prepared.arguments - assert not any("output.schema.json" in upload for upload in prepared.uploads) - assert "/sandbox/oar-runtime/extensions/oar-validate-tools.ts" in ( - prepared.arguments + for task_id in ("review-document", "review-repository"): + input_name = "document.txt" if task_id == "review-document" else "repository" + input_path = f"/workspace/input/{input_name}" + prepared = prepare_resources( + resolved, + task_id, + { + "focus": "Focus on authentication.", + "context": "Pre-release review.", + "oar.input_name": input_name, + "oar.input_path": input_path, + }, ) - finally: - prepared.close() + try: + assert "submit_result" not in prepared.arguments + assert not any( + "output.schema.json" in upload for upload in prepared.uploads + ) + assert "/sandbox/oar-runtime/extensions/oar-validate-tools.ts" in ( + prepared.arguments + ) + prompt_upload = next( + item + for item in prepared.uploads + if item.endswith(":/sandbox/oar-runtime/prompt.md") + ) + prompt = Path(prompt_upload.rpartition(":")[0]).read_text() + assert input_path in prompt + assert "Focus on authentication." in prompt + assert "Pre-release review." in prompt + assert "{{" not in prompt + finally: + prepared.close() def test_custom_extension_and_declared_tool_are_staged(tmp_path: Path) -> None: @@ -143,12 +167,21 @@ def test_custom_extension_and_declared_tool_are_staged(tmp_path: Path) -> None: extension_path = profile / "custom-check.ts" extension_path.write_text("export default function () {}\n") document = yaml.safe_load((profile / "profile.yaml").read_text()) - task = document["tasks"]["review"] + task = document["tasks"]["review-document"] task["tools"].append("custom_check") task["extensions"] = [{"path": "custom-check.ts", "tools": ["custom_check"]}] (profile / "profile.yaml").write_text(yaml.safe_dump(document, sort_keys=False)) - prepared = prepare_resources(load_profile(profile), "review") + prepared = prepare_resources( + load_profile(profile), + "review-document", + { + "focus": "Review the complete document.", + "context": "No additional context was provided.", + "oar.input_name": "document.md", + "oar.input_path": "/workspace/input/document.md", + }, + ) try: assert "/sandbox/oar-runtime/extensions/00-custom-check.ts" in ( prepared.arguments diff --git a/projects/openshell-agent-runner/tests/test_cli.py b/projects/openshell-agent-runner/tests/test_cli.py index 351bc72..658e6f6 100644 --- a/projects/openshell-agent-runner/tests/test_cli.py +++ b/projects/openshell-agent-runner/tests/test_cli.py @@ -87,6 +87,7 @@ def test_run_help_has_only_the_supported_override_surface() -> None: "--task", "--output", "--input", + "--prompt-var", "--upload", "--env", "--gateway", @@ -123,17 +124,20 @@ def test_doctor_separates_native_output_with_blank_lines(monkeypatch) -> None: def test_run_help_describes_selected_profile_task() -> None: result = CliRunner().invoke( app, - ["run", str(PACKAGED_PROFILE), "--task", "review", "--help"], + ["run", str(PACKAGED_PROFILE), "--task", "review-document", "--help"], ) assert result.exit_code == 0 - assert "reviewer:review" in result.stdout + assert "reviewer:review-document" in result.stdout assert ( "Review an input document and return a useful written result." in result.stdout ) assert "--input DOCUMENT" in result.stdout assert "Required argument:" in result.stdout assert "Host document to review." in result.stdout + assert "--prompt-var focus=VALUE" in result.stdout + assert "--prompt-var context=VALUE" in result.stdout + assert "Default: Review the complete document." in result.stdout assert "Additional configured uploads:" in result.stdout assert "Configured environment:" in result.stdout assert "None. Add values with --env KEY=VALUE." in result.stdout @@ -142,15 +146,31 @@ def test_run_help_describes_selected_profile_task() -> None: assert "Options" not in result.stdout +def test_run_help_describes_repository_input() -> None: + result = CliRunner().invoke( + app, + ["run", str(PACKAGED_PROFILE), "--task", "review-repository", "--help"], + ) + + assert result.exit_code == 0 + assert "reviewer:review-repository" in result.stdout + assert "Review an input code repository" in result.stdout + assert "--input REPOSITORY" in result.stdout + assert "Host code repository to review." in result.stdout + assert "--prompt-var focus=VALUE" in result.stdout + assert "--prompt-var context=VALUE" in result.stdout + assert "Default: Review the entire repository." in result.stdout + + def test_run_help_colors_selected_profile_task() -> None: result = CliRunner().invoke( app, - ["run", str(PACKAGED_PROFILE), "--task", "review", "--help"], + ["run", str(PACKAGED_PROFILE), "--task", "review-document", "--help"], color=True, ) assert result.exit_code == 0 - assert "\x1b[36m\x1b[1mreviewer:review\x1b[0m" in result.stdout + assert "\x1b[36m\x1b[1mreviewer:review-document\x1b[0m" in result.stdout assert "\x1b[33m\x1b[1mUsage:\x1b[0m" in result.stdout assert "\x1b[32m oar run " in result.stdout @@ -177,7 +197,7 @@ def test_run_dry_run_does_not_publish_output(tmp_path: Path) -> None: "run", str(PACKAGED_PROFILE), "--task", - "review", + "review-document", "--output", str(output), "--input", @@ -205,7 +225,7 @@ def test_document_task_requires_input() -> None: "run", str(PACKAGED_PROFILE), "--task", - "review", + "review-document", "--output", "review.json", "--dry-run", @@ -216,6 +236,65 @@ def test_document_task_requires_input() -> None: assert "requires --input DOCUMENT" in result.stderr +def test_repository_task_uploads_directory_and_sets_working_directory( + tmp_path: Path, +) -> None: + output = tmp_path / "review.md" + repository = tmp_path / "source-repository" + repository.mkdir() + result = CliRunner().invoke( + app, + [ + "run", + str(PACKAGED_PROFILE), + "--task", + "review-repository", + "--output", + str(output), + "--input", + str(repository), + "--prompt-var", + "focus=src/auth and tests/auth", + "--prompt-var", + "context=Pre-release review", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert f"{repository.resolve()} /workspace/input" in result.stdout + assert "--env REPOSITORY_ROOT=/workspace/input/source-repository" in result.stdout + assert not output.exists() + + +def test_repository_task_requires_input() -> None: + result = CliRunner().invoke( + app, + [ + "run", + str(PACKAGED_PROFILE), + "--task", + "review-repository", + "--output", + "review.md", + "--dry-run", + ], + ) + + assert result.exit_code == 2 + assert "requires --input REPOSITORY" in result.stderr + + +def test_removed_review_task_is_unknown() -> None: + result = CliRunner().invoke( + app, + ["run", str(PACKAGED_PROFILE), "--task", "review", "--help"], + ) + + assert result.exit_code == 2 + assert "unknown task 'review'" in result.stderr + + def test_validate_reports_invalid_encoding_as_cli_input_error(tmp_path: Path) -> None: profile = tmp_path / "profile.yaml" profile.write_bytes(b"\xff\xfe") diff --git a/projects/openshell-agent-runner/tests/test_config.py b/projects/openshell-agent-runner/tests/test_config.py index 1118203..e35a76f 100644 --- a/projects/openshell-agent-runner/tests/test_config.py +++ b/projects/openshell-agent-runner/tests/test_config.py @@ -28,7 +28,40 @@ def test_packaged_profile_validates() -> None: assert resolved.profile.id == "reviewer" assert resolved.runtime.model == "MODEL_ID" assert resolved.runtime.thinking == "high" - assert resolved.profile.tasks["review"].required_input == "document" + assert list(resolved.profile.tasks) == ["review-document", "review-repository"] + assert resolved.profile.tasks["review-document"].required_input == "document" + assert resolved.profile.tasks["review-repository"].required_input == "repository" + + +def test_unknown_required_input_is_rejected(tmp_path: Path) -> None: + _write_profile(tmp_path, task="required_input: archive") + + with pytest.raises(ConfigurationError, match="required_input"): + load_profile(tmp_path) + + +def test_prompt_variables_must_be_declared_and_used(tmp_path: Path) -> None: + _write_profile( + tmp_path, + task="""prompt_variables: + focus: + description: Review focus.""", + ) + + with pytest.raises(ConfigurationError, match="unused.*focus"): + load_profile(tmp_path) + + (tmp_path / "prompt.md").write_text("Review {{ unknown }}.\n") + with pytest.raises(ConfigurationError, match="unknown.*unknown"): + load_profile(tmp_path) + + +def test_input_prompt_builtins_require_a_task_input(tmp_path: Path) -> None: + _write_profile(tmp_path) + (tmp_path / "prompt.md").write_text("Review {{ oar.input_path }}.\n") + + with pytest.raises(ConfigurationError, match="unknown.*oar.input_path"): + load_profile(tmp_path) def test_profile_argument_must_be_a_directory(tmp_path: Path) -> None: diff --git a/projects/openshell-agent-runner/tests/test_profile_init.py b/projects/openshell-agent-runner/tests/test_profile_init.py index 884bb8f..6ce0694 100644 --- a/projects/openshell-agent-runner/tests/test_profile_init.py +++ b/projects/openshell-agent-runner/tests/test_profile_init.py @@ -44,6 +44,9 @@ def test_omitting_profile_initializes_every_packaged_profile(tmp_path: Path) -> resolved = load_profile(reviewer) assert resolved.runtime.model == "provider/model" assert resolved.runtime.thinking == "medium" + assert list(resolved.profile.tasks) == ["review-document", "review-repository"] + assert (reviewer / "prompt-document.md").is_file() + assert (reviewer / "prompt-repository.md").is_file() models = json.loads((reviewer / "models.json").read_text()) model = models["providers"]["openshell"]["models"][0] assert model == {"id": "provider/model", "reasoning": True} diff --git a/projects/openshell-agent-runner/tests/test_prompt_templates.py b/projects/openshell-agent-runner/tests/test_prompt_templates.py new file mode 100644 index 0000000..1806d22 --- /dev/null +++ b/projects/openshell-agent-runner/tests/test_prompt_templates.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from openshell_agent_runner.prompt_templates import ( + render_prompt_template, + validate_prompt_template, +) + + +def test_multiple_prompt_variables_are_rendered_literally() -> None: + template = "Path: {{ oar.input_path }}\nFocus: {{ focus }}\nContext: {{context}}\n" + values = { + "oar.input_path": "/workspace/input/repository", + "focus": "src/auth and tests/auth", + "context": "Use $HOME and `rm` as literal text.", + } + + assert render_prompt_template(template, values) == ( + "Path: /workspace/input/repository\n" + "Focus: src/auth and tests/auth\n" + "Context: Use $HOME and `rm` as literal text.\n" + ) + + +def test_prompt_template_validation_rejects_unknown_and_unused_variables() -> None: + with pytest.raises(ValueError, match="unknown.*missing"): + validate_prompt_template("{{ missing }}", set(), set()) + with pytest.raises(ValueError, match="unused.*context"): + validate_prompt_template("{{ focus }}", {"focus", "context"}, set()) + + +def test_prompt_template_rejects_missing_values_and_malformed_placeholders() -> None: + with pytest.raises(ValueError, match="missing.*focus"): + render_prompt_template("{{ focus }}", {}) + with pytest.raises(ValueError, match="malformed"): + render_prompt_template("{{ focus-name }}", {"focus-name": "value"}) diff --git a/projects/openshell-agent-runner/tests/test_resolution.py b/projects/openshell-agent-runner/tests/test_resolution.py index a36041e..c410c01 100644 --- a/projects/openshell-agent-runner/tests/test_resolution.py +++ b/projects/openshell-agent-runner/tests/test_resolution.py @@ -1,16 +1,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import shutil from collections.abc import Sequence from pathlib import Path import pytest +import yaml from openshell_agent_runner.errors import ConfigurationError from openshell_agent_runner.runner import RunRequest, resolve_run REPOSITORY = Path(__file__).resolve().parents[3] PROFILE = REPOSITORY / ".github/openshell-agents/profiles/dev-note-reviewer" +PACKAGED_PROFILE = ( + REPOSITORY + / "projects/openshell-agent-runner/src/openshell_agent_runner/profiles/reviewer" +) def request( @@ -29,6 +35,23 @@ def request( ) +def review_request( + task_id: str, + input_path: Path | None, + *, + environments: Sequence[str] = (), + prompt_variables: Sequence[str] = (), +) -> RunRequest: + return RunRequest( + profile_directory=PACKAGED_PROFILE, + task_id=task_id, + output=Path("/tmp/review.md"), + input_path=input_path, + environments=environments, + prompt_variables=prompt_variables, + ) + + def test_native_upload_and_environment_are_forwarded_exactly() -> None: resolved = resolve_run( request( @@ -78,6 +101,184 @@ def test_environment_names_are_forwarded_to_native_openshell() -> None: assert "KEYBOARD_LAYOUT=us" in resolved.environments +@pytest.mark.parametrize( + ("filename", "sandbox_filename"), + [ + ("proposal.md", "document.md"), + ("notes.txt", "document.txt"), + ("config.json", "document.json"), + ("README", "document"), + ], +) +def test_document_input_preserves_ordinary_file_extension( + tmp_path: Path, filename: str, sandbox_filename: str +) -> None: + document = tmp_path / filename + document.write_text("Review me.\n") + + resolved = resolve_run(review_request("review-document", document)) + + assert resolved.uploads == ( + f"{document.resolve()}:/workspace/input/{sandbox_filename}", + ) + assert resolved.environments == ("REPOSITORY_ROOT=/workspace/input",) + assert resolved.input is not None + assert resolved.input.source == document.resolve() + assert resolved.input.sandbox_path == f"/workspace/input/{sandbox_filename}" + assert resolved.input.name == filename + assert dict(resolved.prompt_variables)["oar.input_path"] == ( + f"/workspace/input/{sandbox_filename}" + ) + assert dict(resolved.prompt_variables)["oar.input_name"] == filename + + +def test_repository_input_is_uploaded_and_used_as_working_directory( + tmp_path: Path, +) -> None: + repository = tmp_path / "example-project" + repository.mkdir() + + resolved = resolve_run(review_request("review-repository", repository)) + + assert resolved.uploads == (f"{repository.resolve()}:/workspace/input",) + assert resolved.environments == ( + "REPOSITORY_ROOT=/workspace/input/example-project", + ) + assert resolved.input is not None + assert resolved.input.source == repository.resolve() + assert resolved.input.sandbox_path == "/workspace/input/example-project" + assert resolved.input.name == "example-project" + assert dict(resolved.prompt_variables)["oar.input_path"] == ( + "/workspace/input/example-project" + ) + + +def test_multiple_prompt_variables_override_task_defaults(tmp_path: Path) -> None: + repository = tmp_path / "repository" + repository.mkdir() + + resolved = resolve_run( + review_request( + "review-repository", + repository, + prompt_variables=( + "focus=src/auth and tests/auth", + "context=Pre-release security review", + ), + ) + ) + + values = dict(resolved.prompt_variables) + assert values["focus"] == "src/auth and tests/auth" + assert values["context"] == "Pre-release security review" + + +@pytest.mark.parametrize( + ("prompt_variables", "message"), + [ + (("focus=one", "focus=two"), "duplicate prompt variable"), + (("unknown=value",), "undeclared prompt variable"), + (("oar.input_path=value",), "reserved oar namespace"), + (("bad-name=value",), "invalid prompt variable name"), + (("focus=",), "non-empty NAME=VALUE"), + ], +) +def test_invalid_prompt_variable_assignments_are_rejected( + tmp_path: Path, prompt_variables: tuple[str, ...], message: str +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + + with pytest.raises(ConfigurationError, match=message): + resolve_run( + review_request( + "review-repository", + repository, + prompt_variables=prompt_variables, + ) + ) + + +def test_prompt_variable_without_default_is_required(tmp_path: Path) -> None: + profile = tmp_path / "reviewer" + shutil.copytree(PACKAGED_PROFILE, profile) + document = yaml.safe_load((profile / "profile.yaml").read_text()) + del document["tasks"]["review-repository"]["prompt_variables"]["context"]["default"] + (profile / "profile.yaml").write_text(yaml.safe_dump(document, sort_keys=False)) + repository = tmp_path / "repository" + repository.mkdir() + + with pytest.raises(ConfigurationError, match="missing required.*context"): + resolve_run( + RunRequest( + profile_directory=profile, + task_id="review-repository", + output=Path("/tmp/review.md"), + input_path=repository, + ) + ) + + +@pytest.mark.parametrize( + ("task_id", "input_kind", "message"), + [ + ("review-document", "directory", "input document must be a file"), + ("review-repository", "file", "input repository must be a directory"), + ], +) +def test_required_input_type_is_enforced( + tmp_path: Path, task_id: str, input_kind: str, message: str +) -> None: + input_path = tmp_path / "input" + if input_kind == "directory": + input_path.mkdir() + else: + input_path.write_text("content\n") + + with pytest.raises(ConfigurationError, match=message): + resolve_run(review_request(task_id, input_path)) + + +@pytest.mark.parametrize( + ("task_id", "input_label"), + [ + ("review-document", "DOCUMENT"), + ("review-repository", "REPOSITORY"), + ], +) +def test_required_input_must_be_provided(task_id: str, input_label: str) -> None: + with pytest.raises(ConfigurationError, match=f"requires --input {input_label}"): + resolve_run(review_request(task_id, None)) + + +def test_task_without_required_input_rejects_input(tmp_path: Path) -> None: + document = tmp_path / "document.md" + document.write_text("content\n") + + with pytest.raises(ConfigurationError, match="does not accept --input"): + resolve_run( + RunRequest( + profile_directory=PROFILE, + task_id="editorial", + output=Path("/tmp/review.json"), + input_path=document, + ) + ) + + +def test_required_input_repository_root_cannot_be_overridden(tmp_path: Path) -> None: + repository = tmp_path / "repository" + repository.mkdir() + request_with_override = review_request( + "review-repository", + repository, + environments=("REPOSITORY_ROOT=/workspace/other",), + ) + + with pytest.raises(ConfigurationError, match="conflicting environment values"): + resolve_run(request_with_override) + + @pytest.mark.parametrize( ("environment", "message"), [ From 2aabaddc150c3c63dc8560210899e1692f0fc0f8 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 23 Aug 2026 23:22:41 +0000 Subject: [PATCH 2/5] docs(oar): document reviewer inputs and prompt variables --- projects/openshell-agent-runner/README.md | 48 ++++++++++-- projects/openshell-agent-runner/docs/index.md | 75 ++++++++++++++++++- 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/projects/openshell-agent-runner/README.md b/projects/openshell-agent-runner/README.md index 8684954..e118a29 100644 --- a/projects/openshell-agent-runner/README.md +++ b/projects/openshell-agent-runner/README.md @@ -35,7 +35,7 @@ printf '# Review me\n\nA short document.\n' > document.md uvx --from openshell-agent-runner oar validate ./profiles/reviewer uvx --from openshell-agent-runner oar run ./profiles/reviewer \ - --task review \ + --task review-document \ --gateway openshell \ --input document.md \ --output /tmp/oar-review.md \ @@ -58,7 +58,7 @@ for each run, such as the task, inputs, output path, gateway, and workspace. ```yaml id: reviewer -description: Review an uploaded document. +description: Review an uploaded document or code repository. sandbox: policy: policy.yaml @@ -66,9 +66,29 @@ sandbox: env: [] tasks: - review: + review-document: required_input: document - prompt: prompt.md + prompt: prompt-document.md + prompt_variables: + focus: + description: Areas of the document that deserve special attention. + default: Review the complete document. + context: + description: Additional context that should inform the review. + default: No additional context was provided. + tools: [read, grep, find, ls, bash] + skills: [] + extensions: [] + review-repository: + required_input: repository + prompt: prompt-repository.md + prompt_variables: + focus: + description: Files or directories that deserve special attention. + default: Review the entire repository. + context: + description: Additional context that should inform the review. + default: No additional context was provided. tools: [read, grep, find, ls, bash] skills: [] extensions: [] @@ -93,6 +113,12 @@ schemas, and tools that are not built in or declared by a referenced extension. The runtime also verifies that Pi actually registered every selected tool before the first model request. +Prompts support literal runtime substitution. Tasks declare named +`prompt_variables` with optional defaults, and callers override them with a +repeatable `--prompt-var NAME=VALUE`. Variables without defaults are required. +OAR also supplies reserved input metadata such as `{{ oar.input_path }}` and +`{{ oar.input_name }}`. Templates do not execute expressions or shell syntax. + Add `output_schema` to a task when its result must be JSON. OAR exposes the built-in Pi `submit_result` extension for that task, lets Pi correct invalid submissions during the session, and validates the downloaded result against the @@ -117,7 +143,19 @@ profile and task before `--help`: ```bash uvx --from openshell-agent-runner oar run \ - ./profiles/reviewer --task review --help + ./profiles/reviewer --task review-document --help +``` + +The reviewer also accepts a code repository directory: + +```bash +uvx --from openshell-agent-runner oar run ./profiles/reviewer \ + --task review-repository \ + --gateway openshell \ + --input ./my-project \ + --prompt-var focus="src/auth and tests/auth" \ + --prompt-var context="Pre-release security review" \ + --output /tmp/oar-repository-review.md ``` ## Documentation diff --git a/projects/openshell-agent-runner/docs/index.md b/projects/openshell-agent-runner/docs/index.md index 26f3d93..0bc0e80 100644 --- a/projects/openshell-agent-runner/docs/index.md +++ b/projects/openshell-agent-runner/docs/index.md @@ -45,7 +45,7 @@ printf '# Review me\n\nA short document.\n' > document.md uvx --from openshell-agent-runner oar validate ./profiles/reviewer uvx --from openshell-agent-runner oar run ./profiles/reviewer \ - --task review \ + --task review-document \ --gateway openshell \ --input document.md \ --output /tmp/oar-review.md \ @@ -103,8 +103,13 @@ The CLI supplies run-specific values: - `--task` selects a task from `profile.yaml`. - `--upload SOURCE:DESTINATION` uploads a file or directory using OpenShell's native mapping format. It may be repeated. -- `--input FILE` is an optional document-task convenience. OAR uploads the file - to `/workspace/input/document.md` and sets `REPOSITORY_ROOT=/workspace/input`. +- `--input PATH` supplies the file or directory required by the selected task. + A `document` input is uploaded beneath `/workspace/input` with its ordinary + file extension preserved. A `repository` input is uploaded beneath the same + directory. OAR sets `REPOSITORY_ROOT` to the resulting document or repository + directory. +- `--prompt-var NAME=VALUE` supplies a non-secret runtime prompt variable. It + may be repeated for tasks that declare more than one variable. - `--env KEY=VALUE` adds a sandbox environment value. - `--gateway` selects an existing OpenShell gateway. - `--workspace` selects a gateway-side OpenShell namespace. It defaults to @@ -116,6 +121,47 @@ Environment keys start with a letter or underscore and contain only letters, digits, and underscores. They cannot start with OpenShell's reserved `OPENSHELL_` prefix. +### Prompt variables + +Tasks can declare string variables used by their prompt template: + +```yaml +tasks: + review-repository: + required_input: repository + prompt: prompt-repository.md + prompt_variables: + focus: + description: Files or directories that deserve special attention. + default: Review the entire repository. + context: + description: Additional context that should inform the review. +``` + +Variables with defaults are optional; variables without defaults are required. +Callers can supply several independent values by repeating the option: + +```bash +--prompt-var focus="src/auth and tests/auth" \ +--prompt-var context="Pre-release security review" +``` + +Templates reference declared variables by name and OAR metadata through the +reserved `oar` namespace: + +```markdown +Inspect `{{ oar.input_path }}`, originally provided as +`{{ oar.input_name }}`. + +Focus: {{ focus }} +Context: {{ context }} +``` + +Tasks with required inputs receive `oar.input_path` and `oar.input_name`. +Substitution is literal: prompt templates do not support expressions, +conditionals, loops, or shell evaluation. Unknown, duplicated, missing, unused, +and malformed variables are rejected before the sandbox starts. + ### Tools and extensions Each task lists the tools Pi may use. OAR accepts Pi's built-in `bash`, `edit`, @@ -185,12 +231,33 @@ OpenShell treats a directory destination like `cp`: it creates the source directory beneath that destination. Uploads run in declaration order, so more than one source can intentionally merge into the same destination. +The packaged reviewer uses task-specific required inputs: + +```bash +oar run ./profiles/reviewer \ + --task review-document \ + --input ./document.md \ + --output ./document-review.md + +oar run ./profiles/reviewer \ + --task review-repository \ + --input ./repository \ + --prompt-var focus="src/auth and tests/auth" \ + --prompt-var context="Pre-release security review" \ + --output ./repository-review.md +``` + +Document tasks require a file and repository tasks require a directory. For a +repository task, OAR makes the uploaded repository the agent's working +directory. The repository is an uploaded snapshot; changes inside the sandbox +are disposable and are not synchronized back to the host. + Uploads come from three places: | Source | Contents | | --- | --- | | Profile | `sandbox.upload` mappings shared by every run | -| CLI | Repeatable `--upload` mappings and the optional `--input` document | +| CLI | Repeatable `--upload` mappings and the task's required `--input` | | OAR | Prompt, Pi model settings, skills, extensions, and optional schema | Caller uploads normally live under `/workspace`. OAR runtime uploads live under From cfac2322f8f1ed75869c0a153ecddcf7261fafb5 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 23 Aug 2026 23:36:05 +0000 Subject: [PATCH 3/5] test(oar): cover prompt templating end to end --- .../tests/test_lifecycle.py | 57 ++++++++++++++++++- .../tests/test_prompt_templates.py | 29 ++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/projects/openshell-agent-runner/tests/test_lifecycle.py b/projects/openshell-agent-runner/tests/test_lifecycle.py index a1f47c2..1c825c3 100644 --- a/projects/openshell-agent-runner/tests/test_lifecycle.py +++ b/projects/openshell-agent-runner/tests/test_lifecycle.py @@ -68,7 +68,11 @@ def fake_openshell(tmp_path: Path) -> tuple[Path, Path, Path]: if os.environ.get("FAKE_FAIL_CREATE") == "1": sys.exit(1) if os.environ.get("FAKE_SLEEP_CREATE") == "1": import time; time.sleep(5) -elif operation in {"upload", "exec"}: +elif operation == "upload": + if args[4] == "/sandbox/oar-runtime/prompt.md": + capture = log.with_name("uploaded-prompt.md") + capture.write_text(pathlib.Path(args[3]).read_text()) +elif operation == "exec": pass elif operation == "get": if not state.exists(): sys.exit(1) @@ -131,6 +135,57 @@ def test_create_download_owned_delete_order(tmp_path: Path, monkeypatch) -> None assert operations[-4:] == ["exec", "download", "get", "delete"] +def test_run_agent_renders_complete_prompt_variable_context( + tmp_path: Path, monkeypatch +) -> None: + profile, executable, state, log = prepare(tmp_path, monkeypatch) + document = tmp_path / "brief.txt" + document.write_text("Review me.\n") + (profile / "prompt.md").write_text( + """Input: {{ oar.input_path }} +Name: {{ oar.input_name }} +Focus one: {{ focus }} +Focus two: {{focus}} +Context: {{ context }} +""" + ) + (profile / "profile.yaml").write_text( + """id: test +description: Fake templated profile. +sandbox: + policy: policy.yaml +tasks: + smoke: + required_input: document + prompt: prompt.md + prompt_variables: + focus: + description: Required review focus. + context: + description: Optional review context. + default: Default context. + output_schema: output.schema.json +""" + ) + focus = "--src/auth\nUnicode: café = {{ context }}" + item = replace( + request(profile, executable, tmp_path / "result.json"), + input_path=document, + prompt_variables=(f"focus={focus}",), + ) + + run_agent(item) + + assert not state.exists() + assert log.with_name("uploaded-prompt.md").read_text() == ( + "Input: /workspace/input/document.txt\n" + "Name: brief.txt\n" + f"Focus one: {focus}\n" + f"Focus two: {focus}\n" + "Context: Default context.\n" + ) + + def test_resolved_command_is_the_create_prefix(tmp_path: Path, monkeypatch) -> None: profile, executable, state, log = prepare(tmp_path, monkeypatch) item = request(profile, executable, tmp_path / "result.json") diff --git a/projects/openshell-agent-runner/tests/test_prompt_templates.py b/projects/openshell-agent-runner/tests/test_prompt_templates.py index 1806d22..656ecac 100644 --- a/projects/openshell-agent-runner/tests/test_prompt_templates.py +++ b/projects/openshell-agent-runner/tests/test_prompt_templates.py @@ -24,6 +24,22 @@ def test_multiple_prompt_variables_are_rendered_literally() -> None: ) +def test_repeated_placeholders_render_complex_values_once() -> None: + template = "First: {{ value }}\nSecond: {{value}}\n" + value = "--flag=a=b\nUnicode: café\nLiteral: {{ oar.input_path }}" + + assert render_prompt_template(template, {"value": value}) == ( + f"First: {value}\nSecond: {value}\n" + ) + + +@pytest.mark.parametrize("template", ["", "No variables here.\n"]) +def test_templates_without_placeholders_are_unchanged(template: str) -> None: + validate_prompt_template(template, set(), set()) + + assert render_prompt_template(template, {}) == template + + def test_prompt_template_validation_rejects_unknown_and_unused_variables() -> None: with pytest.raises(ValueError, match="unknown.*missing"): validate_prompt_template("{{ missing }}", set(), set()) @@ -36,3 +52,16 @@ def test_prompt_template_rejects_missing_values_and_malformed_placeholders() -> render_prompt_template("{{ focus }}", {}) with pytest.raises(ValueError, match="malformed"): render_prompt_template("{{ focus-name }}", {"focus-name": "value"}) + + +@pytest.mark.parametrize( + "template", + [ + "Unclosed {{ focus", + "Unopened focus }}", + "Literal syntax {{ example-name }}", + ], +) +def test_unescaped_double_braces_are_rejected(template: str) -> None: + with pytest.raises(ValueError, match="malformed"): + render_prompt_template(template, {}) From a32a08c1762cd33331a4f7459672c2873ca3df58 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 23 Aug 2026 23:42:26 +0000 Subject: [PATCH 4/5] ci(oar): verify renamed document review task --- .github/workflows/repository-agents.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/repository-agents.yml b/.github/workflows/repository-agents.yml index d59cd63..365df0a 100644 --- a/.github/workflows/repository-agents.yml +++ b/.github/workflows/repository-agents.yml @@ -101,7 +101,7 @@ jobs: printf '# Review me\n\nA short document.\n' > "$RUNNER_TEMP/review-input.md" uvx --from "$wheel" oar run \ "$RUNNER_TEMP/profiles/reviewer" \ - --task review \ + --task review-document \ --input "$RUNNER_TEMP/review-input.md" \ --output "$RUNNER_TEMP/review-output.md" \ --dry-run From dc8ff24a4870a3e204ffae2887347ae69610cbb4 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 23 Aug 2026 23:52:49 +0000 Subject: [PATCH 5/5] fix(oar): preserve caller-visible input name --- .../src/openshell_agent_runner/runner.py | 5 +++-- .../tests/test_resolution.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/projects/openshell-agent-runner/src/openshell_agent_runner/runner.py b/projects/openshell-agent-runner/src/openshell_agent_runner/runner.py index 7855ff4..5046643 100644 --- a/projects/openshell-agent-runner/src/openshell_agent_runner/runner.py +++ b/projects/openshell-agent-runner/src/openshell_agent_runner/runner.py @@ -269,6 +269,7 @@ def _resolve_required_input( raise ConfigurationError( f"task {request.task_id!r} requires --input {input_label}" ) + input_name = request.input_path.absolute().name try: input_path = request.input_path.resolve(strict=True) except OSError as error: @@ -287,7 +288,7 @@ def _resolve_required_input( resolved_input = ResolvedInput( source=input_path, sandbox_path=sandbox_input, - name=input_path.name, + name=input_name, ) return ( resolved_input, @@ -300,7 +301,7 @@ def _resolve_required_input( resolved_input = ResolvedInput( source=input_path, sandbox_path=repository_root, - name=input_path.name, + name=input_name, ) return ( resolved_input, diff --git a/projects/openshell-agent-runner/tests/test_resolution.py b/projects/openshell-agent-runner/tests/test_resolution.py index c410c01..4b6a552 100644 --- a/projects/openshell-agent-runner/tests/test_resolution.py +++ b/projects/openshell-agent-runner/tests/test_resolution.py @@ -153,6 +153,21 @@ def test_repository_input_is_uploaded_and_used_as_working_directory( ) +def test_input_name_preserves_the_caller_visible_symlink_name(tmp_path: Path) -> None: + repository = tmp_path / "actual-project" + repository.mkdir() + input_path = tmp_path / "review-me" + input_path.symlink_to(repository, target_is_directory=True) + + resolved = resolve_run(review_request("review-repository", input_path)) + + assert resolved.input is not None + assert resolved.input.source == repository.resolve() + assert resolved.input.sandbox_path == "/workspace/input/actual-project" + assert resolved.input.name == "review-me" + assert dict(resolved.prompt_variables)["oar.input_name"] == "review-me" + + def test_multiple_prompt_variables_override_task_defaults(tmp_path: Path) -> None: repository = tmp_path / "repository" repository.mkdir()