From d9d08e68afe218e63c65b33008d68804421eb50c Mon Sep 17 00:00:00 2001 From: Christopher Tso <3115311+christso@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:55:10 +1100 Subject: [PATCH 1/2] chore(productivity): move writing evals to repo root --- .gitignore | 2 +- README.md | 2 +- .../write-naturally/assertions/baseline-no-tools.mjs | 0 .../write-naturally/baseline-workspace/.gitignore | 0 .../productivity}/write-naturally/extensions/workspace.mjs | 4 ++-- .../productivity}/write-naturally/promptfooconfig.yaml | 0 .../productivity}/write-naturally/prompts/baseline.md | 0 .../productivity}/write-naturally/prompts/rewrite.md | 0 .../productivity}/write-naturally/tests/cases.yaml | 0 .../write-naturally/tests/general-writing.yaml | 0 .../productivity}/write-naturally/tests/humanizer.yaml | 0 .../productivity}/write-naturally/tests/pr-comments.yaml | 0 .../productivity}/write-naturally/tests/pr-designs.yaml | 0 .../write-naturally/tests/review-outcomes.yaml | 0 .../productivity}/write-naturally/workspace/.gitignore | 0 package.json | 6 +++--- plugins/productivity/README.md | 2 +- tests/unit/plugins/write-naturally-baseline.test.ts | 2 +- 18 files changed, 9 insertions(+), 9 deletions(-) rename {plugins/productivity/evals => evals/productivity}/write-naturally/assertions/baseline-no-tools.mjs (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/baseline-workspace/.gitignore (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/extensions/workspace.mjs (84%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/promptfooconfig.yaml (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/prompts/baseline.md (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/prompts/rewrite.md (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/tests/cases.yaml (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/tests/general-writing.yaml (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/tests/humanizer.yaml (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/tests/pr-comments.yaml (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/tests/pr-designs.yaml (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/tests/review-outcomes.yaml (100%) rename {plugins/productivity/evals => evals/productivity}/write-naturally/workspace/.gitignore (100%) diff --git a/.gitignore b/.gitignore index 28a591b..e39dda5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ node_modules/ coverage/ .env .promptfoo/ -plugins/**/evals/**/outputs/ +evals/**/outputs/ diff --git a/README.md b/README.md index 5e3bade..1de8ddd 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ bun run validate bun test ``` -The productivity plugin includes a writing eval suite. Install development dependencies with `bun install`, then use `bun run eval:validate` or `bun run eval:write-naturally`. Its 28 fixtures are fictional and cover team messages, customer updates, planning notes, reviews and technical designs. The writer and judge run in a read-only workspace with network access disabled inside the model workspace; model-service requests still use the configured Codex provider. The suite never posts reviews. The writer defaults to `gpt-6.1-sol`; the grader defaults to `gpt-5.6-luna`. Use `CODEX_MODEL` and `CODEX_JUDGE_MODEL` to override them separately. Focus on review comments with `bun run eval:write-naturally --filter-metadata group=review-regression --max-concurrency 2`. +The productivity plugin's writing evals live in [evals/productivity/write-naturally](evals/productivity/write-naturally/promptfooconfig.yaml). Install development dependencies with `bun install`, then use `bun run eval:validate` or `bun run eval:write-naturally`. Its 28 fixtures are fictional and cover team messages, customer updates, planning notes, reviews and technical designs. The writer and judge run in a read-only workspace with network access disabled inside the model workspace; model-service requests still use the configured Codex provider. The suite never posts reviews. The writer defaults to `gpt-6.1-sol`; the grader defaults to `gpt-5.6-luna`. Use `CODEX_MODEL` and `CODEX_JUDGE_MODEL` to override them separately. Focus on review comments with `bun run eval:write-naturally --filter-metadata group=review-regression --max-concurrency 2`. Additional coverage lives in the [private eval repository](https://github.com/allagentsdev/agent-plugins-evals-private). diff --git a/plugins/productivity/evals/write-naturally/assertions/baseline-no-tools.mjs b/evals/productivity/write-naturally/assertions/baseline-no-tools.mjs similarity index 100% rename from plugins/productivity/evals/write-naturally/assertions/baseline-no-tools.mjs rename to evals/productivity/write-naturally/assertions/baseline-no-tools.mjs diff --git a/plugins/productivity/evals/write-naturally/baseline-workspace/.gitignore b/evals/productivity/write-naturally/baseline-workspace/.gitignore similarity index 100% rename from plugins/productivity/evals/write-naturally/baseline-workspace/.gitignore rename to evals/productivity/write-naturally/baseline-workspace/.gitignore diff --git a/plugins/productivity/evals/write-naturally/extensions/workspace.mjs b/evals/productivity/write-naturally/extensions/workspace.mjs similarity index 84% rename from plugins/productivity/evals/write-naturally/extensions/workspace.mjs rename to evals/productivity/write-naturally/extensions/workspace.mjs index 64089c7..67de9d5 100644 --- a/plugins/productivity/evals/write-naturally/extensions/workspace.mjs +++ b/evals/productivity/write-naturally/extensions/workspace.mjs @@ -3,10 +3,10 @@ import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; const suiteRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -const pluginRoot = resolve(suiteRoot, '../..'); +const repositoryRoot = resolve(suiteRoot, '../../..'); const workspace = resolve(suiteRoot, 'workspace'); const workspaceAgents = resolve(workspace, '.agents'); -const skillSource = resolve(pluginRoot, 'skills/write-naturally'); +const skillSource = resolve(repositoryRoot, 'plugins/productivity/skills/write-naturally'); const skillTarget = resolve(workspaceAgents, 'skills/write-naturally'); export function workspaceHook(hookName, context) { diff --git a/plugins/productivity/evals/write-naturally/promptfooconfig.yaml b/evals/productivity/write-naturally/promptfooconfig.yaml similarity index 100% rename from plugins/productivity/evals/write-naturally/promptfooconfig.yaml rename to evals/productivity/write-naturally/promptfooconfig.yaml diff --git a/plugins/productivity/evals/write-naturally/prompts/baseline.md b/evals/productivity/write-naturally/prompts/baseline.md similarity index 100% rename from plugins/productivity/evals/write-naturally/prompts/baseline.md rename to evals/productivity/write-naturally/prompts/baseline.md diff --git a/plugins/productivity/evals/write-naturally/prompts/rewrite.md b/evals/productivity/write-naturally/prompts/rewrite.md similarity index 100% rename from plugins/productivity/evals/write-naturally/prompts/rewrite.md rename to evals/productivity/write-naturally/prompts/rewrite.md diff --git a/plugins/productivity/evals/write-naturally/tests/cases.yaml b/evals/productivity/write-naturally/tests/cases.yaml similarity index 100% rename from plugins/productivity/evals/write-naturally/tests/cases.yaml rename to evals/productivity/write-naturally/tests/cases.yaml diff --git a/plugins/productivity/evals/write-naturally/tests/general-writing.yaml b/evals/productivity/write-naturally/tests/general-writing.yaml similarity index 100% rename from plugins/productivity/evals/write-naturally/tests/general-writing.yaml rename to evals/productivity/write-naturally/tests/general-writing.yaml diff --git a/plugins/productivity/evals/write-naturally/tests/humanizer.yaml b/evals/productivity/write-naturally/tests/humanizer.yaml similarity index 100% rename from plugins/productivity/evals/write-naturally/tests/humanizer.yaml rename to evals/productivity/write-naturally/tests/humanizer.yaml diff --git a/plugins/productivity/evals/write-naturally/tests/pr-comments.yaml b/evals/productivity/write-naturally/tests/pr-comments.yaml similarity index 100% rename from plugins/productivity/evals/write-naturally/tests/pr-comments.yaml rename to evals/productivity/write-naturally/tests/pr-comments.yaml diff --git a/plugins/productivity/evals/write-naturally/tests/pr-designs.yaml b/evals/productivity/write-naturally/tests/pr-designs.yaml similarity index 100% rename from plugins/productivity/evals/write-naturally/tests/pr-designs.yaml rename to evals/productivity/write-naturally/tests/pr-designs.yaml diff --git a/plugins/productivity/evals/write-naturally/tests/review-outcomes.yaml b/evals/productivity/write-naturally/tests/review-outcomes.yaml similarity index 100% rename from plugins/productivity/evals/write-naturally/tests/review-outcomes.yaml rename to evals/productivity/write-naturally/tests/review-outcomes.yaml diff --git a/plugins/productivity/evals/write-naturally/workspace/.gitignore b/evals/productivity/write-naturally/workspace/.gitignore similarity index 100% rename from plugins/productivity/evals/write-naturally/workspace/.gitignore rename to evals/productivity/write-naturally/workspace/.gitignore diff --git a/package.json b/package.json index 2fdee77..791ae6b 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,9 @@ "scripts": { "validate": "bun run scripts/validate-catalogue.ts", "test": "bun test", - "eval:validate": "promptfoo validate config -c plugins/productivity/evals/write-naturally/promptfooconfig.yaml", - "eval:write-naturally": "promptfoo eval -c plugins/productivity/evals/write-naturally/promptfooconfig.yaml --no-cache --no-share", - "eval:write-naturally:baseline": "bun run scripts/eval-writing-baseline.ts plugins/productivity/evals/write-naturally" + "eval:validate": "promptfoo validate config -c evals/productivity/write-naturally/promptfooconfig.yaml", + "eval:write-naturally": "promptfoo eval -c evals/productivity/write-naturally/promptfooconfig.yaml --no-cache --no-share", + "eval:write-naturally:baseline": "bun run scripts/eval-writing-baseline.ts evals/productivity/write-naturally" }, "devDependencies": { "@openai/codex-sdk": "0.162.1", diff --git a/plugins/productivity/README.md b/plugins/productivity/README.md index ca83b78..f8e80af 100644 --- a/plugins/productivity/README.md +++ b/plugins/productivity/README.md @@ -8,4 +8,4 @@ For example: “So you mean a caller can send text where an integer is required, and validation accepts it?” Once confirmed: “Put that caller example directly in the comment, with what happens and the proposed follow-up.” -The [core skill](skills/write-naturally/SKILL.md) applies to general prose. It loads additional guidance for [reviews](skills/write-naturally/references/reviews.md) or [technical designs](skills/write-naturally/references/technical-designs.md) when needed. The [public eval suite](evals/write-naturally/promptfooconfig.yaml) uses fictional examples, including nontechnical writing. Additional coverage lives in the [private eval repository](https://github.com/allagentsdev/agent-plugins-evals-private). +The [core skill](skills/write-naturally/SKILL.md) applies to general prose. It loads additional guidance for [reviews](skills/write-naturally/references/reviews.md) or [technical designs](skills/write-naturally/references/technical-designs.md) when needed. The [public eval suite](../../evals/productivity/write-naturally/promptfooconfig.yaml) uses fictional examples, including nontechnical writing. Additional coverage lives in the [private eval repository](https://github.com/allagentsdev/agent-plugins-evals-private). diff --git a/tests/unit/plugins/write-naturally-baseline.test.ts b/tests/unit/plugins/write-naturally-baseline.test.ts index b4bbc11..c28314a 100644 --- a/tests/unit/plugins/write-naturally-baseline.test.ts +++ b/tests/unit/plugins/write-naturally-baseline.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'bun:test'; -import baselineNoTools from '../../../plugins/productivity/evals/write-naturally/assertions/baseline-no-tools.mjs'; +import baselineNoTools from '../../../evals/productivity/write-naturally/assertions/baseline-no-tools.mjs'; const context = (items: unknown[]) => ({ providerResponse: { raw: JSON.stringify({ items }) } }); const response = { type: 'agent_message', text: 'Rewritten text.' }; From 6d1bc3c89010842e6ed1c1e7ed5b74074a907762 Mon Sep 17 00:00:00 2001 From: Christopher Tso <3115311+christso@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:56:36 +1100 Subject: [PATCH 2/2] docs(productivity): move writing README into its skill --- plugins/productivity/{ => skills/write-naturally}/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename plugins/productivity/{ => skills/write-naturally}/README.md (64%) diff --git a/plugins/productivity/README.md b/plugins/productivity/skills/write-naturally/README.md similarity index 64% rename from plugins/productivity/README.md rename to plugins/productivity/skills/write-naturally/README.md index f8e80af..cb4348f 100644 --- a/plugins/productivity/README.md +++ b/plugins/productivity/skills/write-naturally/README.md @@ -1,6 +1,6 @@ -# Productivity +# Write naturally -## Write naturally: practical gotchas +## Practical gotchas - **Long conversations:** an agent can read the skill early and later drift from it. Before the final version, check which guidance it is applying. If the answer is vague or the prose has drifted, ask it to reread the skill and the relevant reference. A claim that it remembers the skill is weaker evidence than a fresh read. - **Use your paraphrase:** if you need to ask “so you mean …?”, give the agent your plain-language interpretation, confirm the meaning, then ask it to put that explanation into the text. The next reader should understand it without repeating your clarification. @@ -8,4 +8,4 @@ For example: “So you mean a caller can send text where an integer is required, and validation accepts it?” Once confirmed: “Put that caller example directly in the comment, with what happens and the proposed follow-up.” -The [core skill](skills/write-naturally/SKILL.md) applies to general prose. It loads additional guidance for [reviews](skills/write-naturally/references/reviews.md) or [technical designs](skills/write-naturally/references/technical-designs.md) when needed. The [public eval suite](../../evals/productivity/write-naturally/promptfooconfig.yaml) uses fictional examples, including nontechnical writing. Additional coverage lives in the [private eval repository](https://github.com/allagentsdev/agent-plugins-evals-private). +The [core skill](SKILL.md) applies to general prose. It loads additional guidance for [reviews](references/reviews.md) or [technical designs](references/technical-designs.md) when needed. The [public eval suite](../../../../evals/productivity/write-naturally/promptfooconfig.yaml) uses fictional examples, including nontechnical writing. Additional coverage lives in the [private eval repository](https://github.com/allagentsdev/agent-plugins-evals-private).