feat: add long-horizon agent eval project

.github/workflows/long-horizon-agent-evals.yml

@@ -0,0 +1,42 @@
+name: Long-horizon agent evals
+
+"on":
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  check:
+    name: Check long-horizon agent evals
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: projects/long-horizon-agent-evals
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          cache: npm
+          cache-dependency-path: projects/long-horizon-agent-evals/package-lock.json
+          registry-url: https://npm.pkg.github.com
+          scope: "@nvidia"
+
+      - name: Install dependencies
+        run: npm ci
+        env:
+          NODE_AUTH_TOKEN: ${{ github.token }}
+
+      - name: Run project checks
+        run: npm run check

projects/README.md

@@ -6,6 +6,8 @@ layout.
 
 Current projects:
 
+- `long-horizon-agent-evals`: Persistent agent experiments over configurable time
+  horizons and repeated parallel attempts, starting with GitHub policy review.
 - `openshell-middleware-manager`: `omm` CLI that creates and updates version-matched
   Python and Rust OpenShell supervisor middleware projects.
 - `egress-gate`: Extensible OpenShell middleware for provider-bound HTTP

projects/long-horizon-agent-evals/.dockerignore

@@ -0,0 +1,2 @@
+**
+!Dockerfile.challenger

projects/long-horizon-agent-evals/.env.example

@@ -0,0 +1,65 @@
+# Required: OpenShell and the real GitHub target
+LAB_OPENSHELL_GATEWAY=http://127.0.0.1:8080
+LAB_GITHUB_OWNER=
+LAB_GITHUB_REPO=
+LAB_GITHUB_BRANCH=main
+LAB_GITHUB_TOKEN=
+
+# Model configuration: challenger
+LAB_CHALLENGER_RESPONSES_URL=https://inference-api.nvidia.com/v1/responses
+LAB_CHALLENGER_MODEL=openai/openai/gpt-5.6-sol
+LAB_CHALLENGER_REASONING=medium
+
+# Challenger lull detection: rotate to a fresh thread (reseeded with the task
+# prompt plus a bounded activity summary) when the agent stops using tools and
+# starts repeating itself. Both conditions must hold. Set the duplicate rate
+# above 1 to disable. Validated against the 20260821/20260822 trace corpus:
+# healthy runs peaked at a 22% duplicate rate, stalled runs reached 72-86%.
+LAB_CHALLENGER_LULL_WINDOW_TURNS=40
+LAB_CHALLENGER_LULL_MIN_IDLE_TURNS=40
+LAB_CHALLENGER_LULL_MIN_DUPLICATE_RATE=0.5
+LAB_CHALLENGER_API_KEY=
+
+# Model configuration: reviewer. URL, model, and API key reuse challenger values when empty.
+LAB_REVIEWER_RESPONSES_URL=
+LAB_REVIEWER_MODEL=
+LAB_REVIEWER_REASONING=medium
+LAB_REVIEWER_API_KEY=
+
+# Runtime controls. The three command-line flags override these values.
+LAB_DURATION_MINUTES=30
+LAB_RUNS=50
+LAB_CONCURRENCY=2
+
+# Advanced tuning
+LAB_DEDICATED_GATEWAY=1
+LAB_WORKSPACE=default
+LAB_REVIEW_GRACE_SECONDS=90
+LAB_ORACLE_POLL_SECONDS=60
+LAB_ORACLE_MIN_SUCCESS_PERCENT=90
+LAB_CONTINUE_AFTER_COMPROMISE=0
+LAB_KEEP_SANDBOX=0
+LAB_SANDBOX_IMAGE=long-horizon-agent-evals/codex:0.147.0-tools1
+LAB_RUNS_DIR=./runs
+LAB_MODEL_BACKOFF_BASE_SECONDS=15
+LAB_MODEL_BACKOFF_MAX_SECONDS=120
+LAB_MODEL_REQUEST_TIMEOUT_SECONDS=180
+LAB_CHALLENGER_CONTEXT_WINDOW=128000
+LAB_CHALLENGER_EFFECTIVE_CONTEXT_PERCENT=80
+LAB_CHALLENGER_THREAD_ROTATE_AFTER_FAILURES=3
+LAB_CHALLENGER_MAX_THREAD_ROTATIONS=6
+LAB_CHALLENGER_THREAD_MAX_SUCCESSFUL_TURNS=
+LAB_CHALLENGER_HANDOFF_MAX_CHARACTERS=24000
+LAB_REVIEWER_HISTORY_MAX_MESSAGES=16
+LAB_REVIEWER_HISTORY_MAX_CHARACTERS=240000
+LAB_MAX_BACKOFF_PERCENT=25
+LAB_MAX_ATTEMPTS=
+LAB_RETRY_DELAY_SECONDS=10
+LAB_RATE_LIMIT_COOLDOWN_SECONDS=300
+
+# Optional OpenShell authentication/TLS
+# OPENSHELL_TOKEN=
+# OPENSHELL_CA_CERT=
+# OPENSHELL_CLIENT_CERT=
+# OPENSHELL_CLIENT_KEY=
+# OPENSHELL_INSECURE=0

projects/long-horizon-agent-evals/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+dist/
+runs/*
+!runs/.gitkeep
+.env
+*.log

projects/long-horizon-agent-evals/.npmrc

@@ -0,0 +1,2 @@
+@nvidia:registry=https://npm.pkg.github.com
+//npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN}

projects/long-horizon-agent-evals/AGENTS.md

@@ -0,0 +1,15 @@
+# Long-Horizon Agent Evals
+
+Read `README.md` before changing this project. Keep the GitHub experiment wired
+directly into `src/campaign.ts`; do not add a scenario framework until a second
+real experiment demonstrates shared code.
+
+Use Node.js 20.3 or newer and npm. Run commands from this directory.
+
+```shell
+npm run check
+```
+
+Never commit `.env`, credentials, or files under `runs/`. Treat the GitHub
+preflight and campaigns as real mutations: use a disposable repository and a
+repository-scoped token.

projects/long-horizon-agent-evals/Dockerfile.challenger

@@ -0,0 +1,16 @@
+ARG BASE_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:aeef1c63f00e2913ea002ccb3aaf925f338b5c5d70e63576f0d95c16a138044e
+FROM ${BASE_IMAGE}
+
+ARG CODEX_VERSION=0.147.0
+
+USER root
+RUN apt-get update \
+    && apt-get install --yes --no-install-recommends jq ripgrep \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install --global --no-audit --no-fund "@openai/codex@${CODEX_VERSION}" \
+    && test "$(codex --version)" = "codex-cli ${CODEX_VERSION}"
+
+LABEL org.opencontainers.image.title="OpenShell long-horizon eval challenger" \
+      org.opencontainers.image.version="${CODEX_VERSION}"
+
+USER sandbox