Harden benchmark workflow execution

.github/workflows/benchmark-clawbench.yml

@@ -200,14 +200,14 @@ jobs:
     permissions:
       contents: read
       issues: write
+      pull-requests: write
     concurrency:
       group: benchmark-clawbench-${{ needs.resolve.outputs.pr_number || needs.resolve.outputs.head_sha }}
       cancel-in-progress: false
     env:
       HYPEMAN_API_KEY: ${{ secrets.HYPEMAN_API_KEY }}
       HYPEMAN_BASE_URL: ${{ vars.HYPEMAN_BASE_URL }}
       KERNEL_MCP_BENCHMARK_API_KEY: ${{ secrets.KERNEL_MCP_BENCHMARK_API_KEY }}
-      KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}
       PURELY_MAIL_API_KEY: ${{ secrets.PURELY_MAIL_API_KEY }}
       PURELY_MAIL_DOMAIN: ${{ vars.PURELY_MAIL_DOMAIN }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -231,6 +231,7 @@ jobs:
     steps:
       - name: Mark the PR benchmark as running
         if: needs.resolve.outputs.pr_number != ''
+        continue-on-error: true
         uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
         env:
           PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}
@@ -434,6 +435,7 @@ jobs:
 
       - name: Update PR benchmark comment
         if: always() && needs.resolve.outputs.pr_number != ''
+        continue-on-error: true
         uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
         env:
           PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}
@@ -477,3 +479,4 @@ jobs:
           [[ "$BASELINE_STATUS" == "0" ]]
           [[ "$PUBLISH_OUTCOME" == "success" ]]
           [[ "$REPORT_OUTCOME" == "success" ]]
+          jq -e 'all(.arms[]; .scored > 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null

benchmarks/harbor/README.md

@@ -19,7 +19,7 @@ The image records the current Git SHA, and the generated task records the ClawBe
 - `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1
 - Hypeman CLI credentials
 - a ClawBench checkout containing pinned commit `45a71c4`
-- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project, plus its `KERNEL_PROJECT` name
+- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project; its credential scope is the project source of truth
 - `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for ClawBench account tasks
 - `OPENAI_API_KEY` for Codex, or Anthropic credentials for Claude Code
 - the ClawBench judge variables when using a hosted judge: `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, and `CLAWBENCH_JUDGE_API_TYPE`

benchmarks/harbor/clawbench/prepare-task.py

@@ -52,7 +52,6 @@ def _add_environment(
                     f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}",
                     f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}",
                     'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"',
-                    'KERNEL_PROJECT = "${KERNEL_PROJECT:-}"',
                     'REDIS_URL = "redis://127.0.0.1:6379"',
                 ]
             )

benchmarks/harbor/clawbench/run.sh

@@ -109,7 +109,6 @@ export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL}
   printf 'KERNEL_BASE_URL=%s\n' "$KERNEL_BASE_URL"
   printf 'KERNEL_API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL"
   printf 'API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL"
-  printf 'KERNEL_PROJECT=%s\n' "${KERNEL_PROJECT:-}"
   printf 'PURELY_MAIL_API_KEY=%s\n' "$PURELY_MAIL_API_KEY"
   printf 'PURELY_MAIL_DOMAIN=%s\n' "$PURELY_MAIL_DOMAIN"
   printf 'CLAWBENCH_JUDGE_BASE_URL=%s\n' "${CLAWBENCH_JUDGE_BASE_URL:-}"

benchmarks/harbor/publish-braintrust.ts

@@ -354,6 +354,16 @@ export async function publishBenchmark(
   experimentName: string,
   apiKey: string,
 ): Promise<Record<string, unknown>> {
+  const ungraded = arms
+    .map(summarizeArm)
+    .filter((summary) => summary.scored === 0)
+    .map((summary) => summary.arm);
+  if (ungraded.length > 0) {
+    throw new Error(
+      `Cannot publish benchmark without graded trials for: ${ungraded.join(", ")}`,
+    );
+  }
+
   const api = new BraintrustApi(apiKey);
   const project = await api.request<BraintrustProject>("/v1/project", "POST", {
     name: projectName,

benchmarks/harbor/report.ts

@@ -89,10 +89,16 @@ export function renderMarkdown(
 ): string {
   const lines = ["<!-- kernel-mcp-clawbench -->", `## ${title}`];
   const failed = Object.entries(statuses).filter(([, status]) => status !== 0);
-  if (failed.length > 0) {
+  const ungraded = summaries.filter((summary) => summary.scored === 0);
+  const incomplete = failed.length > 0 || ungraded.length > 0;
+  if (incomplete) {
+    const reasons = [
+      ...failed.map(([arm, status]) => `${arm} exited ${status}`),
+      ...ungraded.map((summary) => `${summary.arm} produced no graded trials`),
+    ];
     lines.push(
       "",
-      `> [!WARNING]\n> Incomplete benchmark: ${failed.map(([arm, status]) => `${arm} exited ${status}`).join(", ")}. Scores below include only completed Harbor results and are not a complete comparison.`,
+      `> [!WARNING]\n> Incomplete benchmark: ${reasons.join(", ")}. Scores below include only completed Harbor results; comparison deltas are suppressed.`,
     );
   }
   lines.push(
@@ -108,7 +114,7 @@ export function renderMarkdown(
 
   const candidate = summaries.find((summary) => summary.arm === "candidate");
   const baseline = summaries.find((summary) => summary.arm === "baseline");
-  if (candidate && baseline && failed.length === 0) {
+  if (candidate && baseline && !incomplete) {
     const signed = (value: number) => {
       const rounded = Number(value.toFixed(3));
       return `${rounded >= 0 ? "+" : ""}${rounded}`;

benchmarks/harbor/results.test.ts

@@ -257,6 +257,16 @@ describe("Harbor result ingestion", () => {
     }
   });
 
+  test("does not publish arms without graded trials", async () => {
+    const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
+    for (const trial of arm.trials) trial.rewards = {};
+    await expect(
+      publishBenchmark([arm], "project", "experiment", "api-key"),
+    ).rejects.toThrow(
+      "Cannot publish benchmark without graded trials for: candidate",
+    );
+  });
+
   test("uses the lenient reward per trial and reports incomplete arms", () => {
     expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({
       key: "reward_lenient",
@@ -288,6 +298,12 @@ describe("Harbor result ingestion", () => {
         { ...summary, arm: "baseline", lenient: 0.2 },
       ]),
     ).toContain("+0.1 lenient");
+    const ungraded = renderMarkdown("test", [
+      { ...summary, arm: "candidate", scored: 0, ungraded: summary.trials },
+      { ...summary, arm: "baseline", scored: 0, ungraded: summary.trials },
+    ]);
+    expect(ungraded).toContain("candidate produced no graded trials");
+    expect(ungraded).not.toContain("Candidate minus baseline");
   });
 
   test("keeps full errors until redaction and clamps derived scores", () => {
@@ -369,6 +385,17 @@ describe("benchmark workflow hardening", () => {
     expect(workflow).not.toContain("baseSha = pull.base.sha");
     expect(workflow).toContain('HARBOR_VERSION: "0.21.0"');
     expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"');
+    expect(workflow).toContain("issues: write\n      pull-requests: write");
+    expect(workflow).not.toContain(
+      "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}",
+    );
+    expect(workflow).toContain("all(.arms[]; .scored > 0)");
+    expect(workflow).toMatch(
+      /- name: Mark the PR benchmark as running\n\s+if:.*\n\s+continue-on-error: true/,
+    );
+    expect(workflow).toMatch(
+      /- name: Update PR benchmark comment\n\s+if:.*\n\s+continue-on-error: true/,
+    );
     expect(workflow).toContain(
       'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")',
     );
@@ -387,7 +414,13 @@ describe("benchmark workflow hardening", () => {
       join(process.cwd(), "benchmarks/harbor/clawbench/verify-task.py"),
       "utf8",
     );
+    const taskPreparer = readFileSync(
+      join(process.cwd(), "benchmarks/harbor/clawbench/prepare-task.py"),
+      "utf8",
+    );
     expect(dockerignore.split("\n")).toContain("*.pem");
+    expect(runner).not.toContain("KERNEL_PROJECT");
+    expect(taskPreparer).not.toContain("KERNEL_PROJECT");
     expect(verifier).toContain('"mcp__kernel__execute_playwright_code"');
     expect(verifier).toContain('"kernel__execute_playwright_code"');
     expect(verifier).toContain('"execute_playwright_code"');