From 44939f5195637a68f600e0ace19038d3029ff2e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 13:13:52 +0000 Subject: [PATCH 1/3] test(skills): the platform skill's evals ask for push notifications on os serve Case 12 in objectstack-platform/evals/config-plugins-ops.json asks to turn on push for an os serve deployment; the expected answer declares requires: ['push'] with a top-level push: key and writes no secret inline. The assertions stay with what SKILL.md teaches today. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01J8VVAjFpbQxLtYHuw3di1T --- .changeset/29969-platform-push-eval.md | 10 ++++++++++ .../evals/config-plugins-ops.json | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 .changeset/29969-platform-push-eval.md diff --git a/.changeset/29969-platform-push-eval.md b/.changeset/29969-platform-push-eval.md new file mode 100644 index 00000000000..39183279c9f --- /dev/null +++ b/.changeset/29969-platform-push-eval.md @@ -0,0 +1,10 @@ +--- +'@objectstack/skills': patch +--- + +test(skills): the platform skill's evals ask for push notifications on an `os serve` deployment + +Clause-②: no +Metadata-protocol: none + +`skills/objectstack-platform/evals/config-plugins-ops.json` gains case 12: the prompt asks to turn on push notifications for an `os serve` deployment, and the expected answer declares `requires: ['push']` with a top-level `push:` key on the stack definition and writes no secret inline (the APNs signing key and the Firebase service-account JSON are named by reference). The assertions stay with what `objectstack-platform/SKILL.md` teaches today, the `push` token and the top-level key; the push block's transport and vendor shape are not asserted until the skill teaches them. diff --git a/skills/objectstack-platform/evals/config-plugins-ops.json b/skills/objectstack-platform/evals/config-plugins-ops.json index 2f68bdc9639..69d7aace1ed 100644 --- a/skills/objectstack-platform/evals/config-plugins-ops.json +++ b/skills/objectstack-platform/evals/config-plugins-ops.json @@ -197,6 +197,24 @@ "PUT /api/v1/meta/permission/crm_sales_rep" ] } + }, + { + "id": 12, + "prompt": "Turn on push notifications for our `os serve` deployment: our own iOS and Android apps, signed with our APNs key and our Firebase project.", + "expected_output": "`requires: ['push']` on the stack root (the token mounts `@objectstack/service-push`) plus a top-level `push:` key on the stack definition for the transport; no host code, no `PushServicePlugin` in `plugins:`. The APNs signing key and the Firebase service-account JSON stay in the secrets layer (an environment variable in the open tier) and the config names them by reference (`credentialRef`): no `.p8` key, `privateKey` or service-account JSON written inline.", + "files": [], + "assertions": { + "must_contain": [ + "requires", + "'push'", + "push:" + ], + "must_not_contain": [ + "BEGIN PRIVATE KEY", + "privateKey:", + "\"private_key\"" + ] + } } ] } From e871e3cdc09f3e72f65baf7bf5eb3183a560b4e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 13:58:15 +0000 Subject: [PATCH 2/3] test(skills): the push eval case pins the direct transport and refuses the relay Case 12's expected answer names transport: 'direct' with apns / fcm blocks, each with its credentialRef, and says the hosted relay (also an absent or empty push block) is refused at boot until its sender ships; the case adds transport: 'relay' to must_not_contain. The changeset says the same. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01J8VVAjFpbQxLtYHuw3di1T --- .changeset/29969-platform-push-eval.md | 2 +- skills/objectstack-platform/evals/config-plugins-ops.json | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.changeset/29969-platform-push-eval.md b/.changeset/29969-platform-push-eval.md index 39183279c9f..7b21645e490 100644 --- a/.changeset/29969-platform-push-eval.md +++ b/.changeset/29969-platform-push-eval.md @@ -7,4 +7,4 @@ test(skills): the platform skill's evals ask for push notifications on an `os se Clause-②: no Metadata-protocol: none -`skills/objectstack-platform/evals/config-plugins-ops.json` gains case 12: the prompt asks to turn on push notifications for an `os serve` deployment, and the expected answer declares `requires: ['push']` with a top-level `push:` key on the stack definition and writes no secret inline (the APNs signing key and the Firebase service-account JSON are named by reference). The assertions stay with what `objectstack-platform/SKILL.md` teaches today, the `push` token and the top-level key; the push block's transport and vendor shape are not asserted until the skill teaches them. +`skills/objectstack-platform/evals/config-plugins-ops.json` gains case 12: the prompt asks to turn on push notifications for an `os serve` deployment of the team's own white-label apps, and the expected answer declares `requires: ['push']` with a top-level `push:` key on the stack definition using `transport: 'direct'` and an `apns` and/or `fcm` block, each naming its secret by `credentialRef`, and writes no secret inline. The case refuses `transport: 'relay'`: the hosted relay, which an absent or empty `push` block also reads as, is refused at boot until its sender ships. diff --git a/skills/objectstack-platform/evals/config-plugins-ops.json b/skills/objectstack-platform/evals/config-plugins-ops.json index 69d7aace1ed..36d1b163db0 100644 --- a/skills/objectstack-platform/evals/config-plugins-ops.json +++ b/skills/objectstack-platform/evals/config-plugins-ops.json @@ -201,7 +201,7 @@ { "id": 12, "prompt": "Turn on push notifications for our `os serve` deployment: our own iOS and Android apps, signed with our APNs key and our Firebase project.", - "expected_output": "`requires: ['push']` on the stack root (the token mounts `@objectstack/service-push`) plus a top-level `push:` key on the stack definition for the transport; no host code, no `PushServicePlugin` in `plugins:`. The APNs signing key and the Firebase service-account JSON stay in the secrets layer (an environment variable in the open tier) and the config names them by reference (`credentialRef`): no `.p8` key, `privateKey` or service-account JSON written inline.", + "expected_output": "`requires: ['push']` on the stack root (the token mounts `@objectstack/service-push`) plus a top-level `push:` key on the stack definition with `transport: 'direct'` and an `apns` and/or `fcm` block, each with its `credentialRef`; the hosted relay (`transport` relay, or an absent or empty `push` block) is refused at boot until its sender ships; no host code, no `PushServicePlugin` in `plugins:`. The APNs signing key and the Firebase service-account JSON stay in the secrets layer (an environment variable in the open tier) and the config names them by reference: no `.p8` key, `privateKey` or service-account JSON written inline.", "files": [], "assertions": { "must_contain": [ @@ -212,7 +212,8 @@ "must_not_contain": [ "BEGIN PRIVATE KEY", "privateKey:", - "\"private_key\"" + "\"private_key\"", + "transport: 'relay'" ] } } From 66bfe707d27053ed0d7790d1f43d3c94a8bdb53c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 13:59:56 +0000 Subject: [PATCH 3/3] Revert "test(skills): the push eval case pins the direct transport and refuses the relay" This reverts commit e871e3cdc09f3e72f65baf7bf5eb3183a560b4e1. Pinning the transport moves to the follow-up that first teaches the push block's shape in the skill; the case stays as reviewed. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01J8VVAjFpbQxLtYHuw3di1T --- .changeset/29969-platform-push-eval.md | 2 +- skills/objectstack-platform/evals/config-plugins-ops.json | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.changeset/29969-platform-push-eval.md b/.changeset/29969-platform-push-eval.md index 7b21645e490..39183279c9f 100644 --- a/.changeset/29969-platform-push-eval.md +++ b/.changeset/29969-platform-push-eval.md @@ -7,4 +7,4 @@ test(skills): the platform skill's evals ask for push notifications on an `os se Clause-②: no Metadata-protocol: none -`skills/objectstack-platform/evals/config-plugins-ops.json` gains case 12: the prompt asks to turn on push notifications for an `os serve` deployment of the team's own white-label apps, and the expected answer declares `requires: ['push']` with a top-level `push:` key on the stack definition using `transport: 'direct'` and an `apns` and/or `fcm` block, each naming its secret by `credentialRef`, and writes no secret inline. The case refuses `transport: 'relay'`: the hosted relay, which an absent or empty `push` block also reads as, is refused at boot until its sender ships. +`skills/objectstack-platform/evals/config-plugins-ops.json` gains case 12: the prompt asks to turn on push notifications for an `os serve` deployment, and the expected answer declares `requires: ['push']` with a top-level `push:` key on the stack definition and writes no secret inline (the APNs signing key and the Firebase service-account JSON are named by reference). The assertions stay with what `objectstack-platform/SKILL.md` teaches today, the `push` token and the top-level key; the push block's transport and vendor shape are not asserted until the skill teaches them. diff --git a/skills/objectstack-platform/evals/config-plugins-ops.json b/skills/objectstack-platform/evals/config-plugins-ops.json index 36d1b163db0..69d7aace1ed 100644 --- a/skills/objectstack-platform/evals/config-plugins-ops.json +++ b/skills/objectstack-platform/evals/config-plugins-ops.json @@ -201,7 +201,7 @@ { "id": 12, "prompt": "Turn on push notifications for our `os serve` deployment: our own iOS and Android apps, signed with our APNs key and our Firebase project.", - "expected_output": "`requires: ['push']` on the stack root (the token mounts `@objectstack/service-push`) plus a top-level `push:` key on the stack definition with `transport: 'direct'` and an `apns` and/or `fcm` block, each with its `credentialRef`; the hosted relay (`transport` relay, or an absent or empty `push` block) is refused at boot until its sender ships; no host code, no `PushServicePlugin` in `plugins:`. The APNs signing key and the Firebase service-account JSON stay in the secrets layer (an environment variable in the open tier) and the config names them by reference: no `.p8` key, `privateKey` or service-account JSON written inline.", + "expected_output": "`requires: ['push']` on the stack root (the token mounts `@objectstack/service-push`) plus a top-level `push:` key on the stack definition for the transport; no host code, no `PushServicePlugin` in `plugins:`. The APNs signing key and the Firebase service-account JSON stay in the secrets layer (an environment variable in the open tier) and the config names them by reference (`credentialRef`): no `.p8` key, `privateKey` or service-account JSON written inline.", "files": [], "assertions": { "must_contain": [ @@ -212,8 +212,7 @@ "must_not_contain": [ "BEGIN PRIVATE KEY", "privateKey:", - "\"private_key\"", - "transport: 'relay'" + "\"private_key\"" ] } }