diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index 4b8f6ca..5753161 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -200,6 +200,7 @@ jobs: permissions: contents: read issues: write + pull-requests: write concurrency: group: benchmark-clawbench-${{ needs.resolve.outputs.pr_number || needs.resolve.outputs.head_sha }} cancel-in-progress: false @@ -207,7 +208,6 @@ jobs: HYPEMAN_API_KEY: ${{ secrets.HYPEMAN_API_KEY }} HYPEMAN_BASE_URL: ${{ vars.HYPEMAN_BASE_URL }} KERNEL_MCP_BENCHMARK_API_KEY: ${{ secrets.KERNEL_MCP_BENCHMARK_API_KEY }} - KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }} PURELY_MAIL_API_KEY: ${{ secrets.PURELY_MAIL_API_KEY }} PURELY_MAIL_DOMAIN: ${{ vars.PURELY_MAIL_DOMAIN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -231,6 +231,7 @@ jobs: steps: - name: Mark the PR benchmark as running if: needs.resolve.outputs.pr_number != '' + continue-on-error: true uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} @@ -434,6 +435,7 @@ jobs: - name: Update PR benchmark comment if: always() && needs.resolve.outputs.pr_number != '' + continue-on-error: true uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} @@ -477,3 +479,4 @@ jobs: [[ "$BASELINE_STATUS" == "0" ]] [[ "$PUBLISH_OUTCOME" == "success" ]] [[ "$REPORT_OUTCOME" == "success" ]] + jq -e 'all(.arms[]; .scored > 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index b2d8c1f..7ef08c4 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -19,7 +19,7 @@ The image records the current Git SHA, and the generated task records the ClawBe - `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1 - Hypeman CLI credentials - a ClawBench checkout containing pinned commit `45a71c4` -- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project, plus its `KERNEL_PROJECT` name +- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project; its credential scope is the project source of truth - `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for ClawBench account tasks - `OPENAI_API_KEY` for Codex, or Anthropic credentials for Claude Code - the ClawBench judge variables when using a hosted judge: `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, and `CLAWBENCH_JUDGE_API_TYPE` diff --git a/benchmarks/harbor/clawbench/prepare-task.py b/benchmarks/harbor/clawbench/prepare-task.py index 5dc0bc8..3539a2f 100755 --- a/benchmarks/harbor/clawbench/prepare-task.py +++ b/benchmarks/harbor/clawbench/prepare-task.py @@ -52,7 +52,6 @@ def _add_environment( f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", 'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', - 'KERNEL_PROJECT = "${KERNEL_PROJECT:-}"', 'REDIS_URL = "redis://127.0.0.1:6379"', ] ) diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index 4f59be5..90fe1cb 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -109,7 +109,6 @@ export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} printf 'KERNEL_BASE_URL=%s\n' "$KERNEL_BASE_URL" printf 'KERNEL_API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL" printf 'API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL" - printf 'KERNEL_PROJECT=%s\n' "${KERNEL_PROJECT:-}" printf 'PURELY_MAIL_API_KEY=%s\n' "$PURELY_MAIL_API_KEY" printf 'PURELY_MAIL_DOMAIN=%s\n' "$PURELY_MAIL_DOMAIN" printf 'CLAWBENCH_JUDGE_BASE_URL=%s\n' "${CLAWBENCH_JUDGE_BASE_URL:-}" diff --git a/benchmarks/harbor/publish-braintrust.ts b/benchmarks/harbor/publish-braintrust.ts index 51b032c..fc342d7 100644 --- a/benchmarks/harbor/publish-braintrust.ts +++ b/benchmarks/harbor/publish-braintrust.ts @@ -354,6 +354,16 @@ export async function publishBenchmark( experimentName: string, apiKey: string, ): Promise> { + const ungraded = arms + .map(summarizeArm) + .filter((summary) => summary.scored === 0) + .map((summary) => summary.arm); + if (ungraded.length > 0) { + throw new Error( + `Cannot publish benchmark without graded trials for: ${ungraded.join(", ")}`, + ); + } + const api = new BraintrustApi(apiKey); const project = await api.request("/v1/project", "POST", { name: projectName, diff --git a/benchmarks/harbor/report.ts b/benchmarks/harbor/report.ts index 55eb752..cb91a6c 100644 --- a/benchmarks/harbor/report.ts +++ b/benchmarks/harbor/report.ts @@ -89,10 +89,16 @@ export function renderMarkdown( ): string { const lines = ["", `## ${title}`]; const failed = Object.entries(statuses).filter(([, status]) => status !== 0); - if (failed.length > 0) { + const ungraded = summaries.filter((summary) => summary.scored === 0); + const incomplete = failed.length > 0 || ungraded.length > 0; + if (incomplete) { + const reasons = [ + ...failed.map(([arm, status]) => `${arm} exited ${status}`), + ...ungraded.map((summary) => `${summary.arm} produced no graded trials`), + ]; lines.push( "", - `> [!WARNING]\n> Incomplete benchmark: ${failed.map(([arm, status]) => `${arm} exited ${status}`).join(", ")}. Scores below include only completed Harbor results and are not a complete comparison.`, + `> [!WARNING]\n> Incomplete benchmark: ${reasons.join(", ")}. Scores below include only completed Harbor results; comparison deltas are suppressed.`, ); } lines.push( @@ -108,7 +114,7 @@ export function renderMarkdown( const candidate = summaries.find((summary) => summary.arm === "candidate"); const baseline = summaries.find((summary) => summary.arm === "baseline"); - if (candidate && baseline && failed.length === 0) { + if (candidate && baseline && !incomplete) { const signed = (value: number) => { const rounded = Number(value.toFixed(3)); return `${rounded >= 0 ? "+" : ""}${rounded}`; diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index dbe80fe..ca2ccc0 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -257,6 +257,16 @@ describe("Harbor result ingestion", () => { } }); + test("does not publish arms without graded trials", async () => { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + for (const trial of arm.trials) trial.rewards = {}; + await expect( + publishBenchmark([arm], "project", "experiment", "api-key"), + ).rejects.toThrow( + "Cannot publish benchmark without graded trials for: candidate", + ); + }); + test("uses the lenient reward per trial and reports incomplete arms", () => { expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({ key: "reward_lenient", @@ -288,6 +298,12 @@ describe("Harbor result ingestion", () => { { ...summary, arm: "baseline", lenient: 0.2 }, ]), ).toContain("+0.1 lenient"); + const ungraded = renderMarkdown("test", [ + { ...summary, arm: "candidate", scored: 0, ungraded: summary.trials }, + { ...summary, arm: "baseline", scored: 0, ungraded: summary.trials }, + ]); + expect(ungraded).toContain("candidate produced no graded trials"); + expect(ungraded).not.toContain("Candidate minus baseline"); }); test("keeps full errors until redaction and clamps derived scores", () => { @@ -369,6 +385,17 @@ describe("benchmark workflow hardening", () => { expect(workflow).not.toContain("baseSha = pull.base.sha"); expect(workflow).toContain('HARBOR_VERSION: "0.21.0"'); expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"'); + expect(workflow).toContain("issues: write\n pull-requests: write"); + expect(workflow).not.toContain( + "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}", + ); + expect(workflow).toContain("all(.arms[]; .scored > 0)"); + expect(workflow).toMatch( + /- name: Mark the PR benchmark as running\n\s+if:.*\n\s+continue-on-error: true/, + ); + expect(workflow).toMatch( + /- name: Update PR benchmark comment\n\s+if:.*\n\s+continue-on-error: true/, + ); expect(workflow).toContain( 'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")', ); @@ -387,7 +414,13 @@ describe("benchmark workflow hardening", () => { join(process.cwd(), "benchmarks/harbor/clawbench/verify-task.py"), "utf8", ); + const taskPreparer = readFileSync( + join(process.cwd(), "benchmarks/harbor/clawbench/prepare-task.py"), + "utf8", + ); expect(dockerignore.split("\n")).toContain("*.pem"); + expect(runner).not.toContain("KERNEL_PROJECT"); + expect(taskPreparer).not.toContain("KERNEL_PROJECT"); expect(verifier).toContain('"mcp__kernel__execute_playwright_code"'); expect(verifier).toContain('"kernel__execute_playwright_code"'); expect(verifier).toContain('"execute_playwright_code"');