diff --git a/.changeset/27455-platform-eval-self-agreement.md b/.changeset/27455-platform-eval-self-agreement.md new file mode 100644 index 00000000000..d1cfe75519e --- /dev/null +++ b/.changeset/27455-platform-eval-self-agreement.md @@ -0,0 +1,12 @@ +--- +'@objectstack/skills': patch +--- + +fix(skills): four objectstack-platform eval cases agree with their own `must_not_contain` assertions + +Clause-②: no +Metadata-protocol: none + +- **Before.** In `skills/objectstack-platform/evals/config-plugins-ops.json`, four cases refused a string their own `expected_output` spelled: eval 1 `tableName`, eval 2 `data:beforeInsert` and `onInstall`, eval 4 `@objectstack/adapter-`, eval 7 `role:` and `admin_full_access`. Graded against those assertions, the reference answer itself scored wrong, so the eval could not tell a good configuration, plugin or operations answer from a bad one. +- **Now.** Judged per case from the skill text. Eval 1 narrows its assertion to the key spelling `tableName:` (the skill's own warning, "never set `tableName`", names the bare word; the wrong form is the key beside `workflows:` / `approvals:`). Eval 2 says "a `data:*` kernel hook registers and never fires" and "no install / enable / disable hooks" instead of spelling the phantom hook and method. Eval 4 says "no `adapter-*` packages exist". Eval 7 names the refused alias as `role` (the key spelling `role:` stays refused) and says "no platform set" (the skill's term for a set such as the refused platform-admin one). No assertion is deleted, no case added or removed; the file shrinks by 13 tokens under its unchanged ceiling. +- No schema, key or behaviour changes. diff --git a/skills/objectstack-platform/evals/config-plugins-ops.json b/skills/objectstack-platform/evals/config-plugins-ops.json index 599f1e690a8..8664aa018f2 100644 --- a/skills/objectstack-platform/evals/config-plugins-ops.json +++ b/skills/objectstack-platform/evals/config-plugins-ops.json @@ -8,13 +8,13 @@ "files": [], "assertions": { "must_contain": ["defineStack", "manifest", "engines", "Object.values("], - "must_not_contain": ["workflows:", "approvals:", "tableName"] + "must_not_contain": ["workflows:", "approvals:", "tableName:"] } }, { "id": 2, "prompt": "Build a plugin registering an `audit` service that records every metadata reload and every `task` insert.", - "expected_output": "A `Plugin` with `name`, `init(ctx)` calling `ctx.registerService('audit', ...)` and `ctx.hook('metadata:reloaded', ...)`, `start(ctx)`, `destroy()`. Record events are not kernel hooks — `ctx.hook('data:beforeInsert')` registers and never fires — so the task insert is a data hook (objectstack-data). No `onInstall` / `onEnable` / `onDisable`.", + "expected_output": "A `Plugin` with `name`, `init(ctx)` calling `ctx.registerService('audit', ...)` and `ctx.hook('metadata:reloaded', ...)`, `start(ctx)`, `destroy()`. Record events are not kernel hooks — a `data:*` kernel hook registers and never fires — so the task insert is a data hook (objectstack-data). No install / enable / disable hooks.", "files": [], "assertions": { "must_contain": ["registerService", "ctx.hook(", "metadata:reloaded", "init(", "destroy"], @@ -34,7 +34,7 @@ { "id": 4, "prompt": "We deploy on Vercel serverless functions in our own Hono app; mount the ObjectStack API under /api.", - "expected_output": "`createHonoApp({ kernel })` from `@objectstack/hono` (prefix defaults to `/api`) in the host app — `HonoServerPlugin` only when ObjectStack owns the server; `LiteKernel` for a cold-start-sensitive host; `/auth/*` follows the auth `basePath`, not `prefix`; no `@objectstack/adapter-*` packages exist.", + "expected_output": "`createHonoApp({ kernel })` from `@objectstack/hono` (prefix defaults to `/api`) in the host app — `HonoServerPlugin` only when ObjectStack owns the server; `LiteKernel` for a cold-start-sensitive host; `/auth/*` follows the auth `basePath`, not `prefix`; no `adapter-*` packages exist.", "files": [], "assertions": { "must_contain": ["createHonoApp", "@objectstack/hono", "LiteKernel"], @@ -64,7 +64,7 @@ { "id": 7, "prompt": "Prepare a demo of our contracts app: the seed assigns matters to 'Legal Counsel 1' and 'Paralegal 1', and the demo opens signed in as Legal Counsel 1.", - "expected_output": "Both go in `demoStaff` — `name` as the seed references them, `email`, a `position` / `permissionSets` the stack declares — and `os dev` creates the accounts with the seed password: no `password` key, no `role:` alias, no platform set like `admin_full_access`, no hand-written `sys_user` seed row. `devLogins` creates nothing (at most a banner entry); `os build` drops `demoStaff`; production creates no demo account.", + "expected_output": "Both go in `demoStaff` — `name` as the seed references them, `email`, a `position` / `permissionSets` the stack declares — and `os dev` creates the accounts with the seed password: no `password` key, no `role` alias, no platform set, no hand-written `sys_user` seed row. `devLogins` creates nothing (at most a banner entry); `os build` drops `demoStaff`; production creates no demo account.", "files": [], "assertions": { "must_contain": ["demoStaff", "position", "permissionSets", "os dev"],